From 66eb7e578fa8cb4b427b24d028418bb10cf62545 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 19 Feb 2026 11:30:15 -0800 Subject: [PATCH 01/82] feat!: rearchitect proxy with Cloudflare Workers-compatible API --- CHANGELOG.md | 166 - CLAUDE.md | 33 + CODE_OF_CONDUCT.md | 132 - CONTRIBUTING.md | 90 - Cargo.lock | 3469 ++++++++++------- Cargo.toml | 106 +- Dockerfile | 48 +- LICENSE | 7 - README.md | 325 +- config.example.toml | 105 + crates/libs/auth/Cargo.toml | 20 + crates/libs/auth/README.md | 74 + crates/libs/auth/src/jwks.rs | 179 + crates/libs/auth/src/lib.rs | 174 + crates/libs/auth/src/sts.rs | 52 + crates/libs/core/Cargo.toml | 36 + crates/libs/core/README.md | 117 + crates/libs/core/src/auth.rs | 295 ++ crates/libs/core/src/backend.rs | 192 + crates/libs/core/src/config/cached.rs | 222 ++ crates/libs/core/src/config/dynamodb.rs | 238 ++ crates/libs/core/src/config/http.rs | 178 + crates/libs/core/src/config/mod.rs | 79 + crates/libs/core/src/config/postgres.rs | 165 + crates/libs/core/src/config/static_file.rs | 165 + crates/libs/core/src/error.rs | 71 + crates/libs/core/src/lib.rs | 28 + crates/libs/core/src/maybe_send.rs | 45 + crates/libs/core/src/proxy.rs | 357 ++ crates/libs/core/src/resolver.rs | 194 + crates/libs/core/src/s3/list_rewrite.rs | 112 + crates/libs/core/src/s3/mod.rs | 3 + crates/libs/core/src/s3/request.rs | 189 + crates/libs/core/src/s3/response.rs | 184 + crates/libs/core/src/stream.rs | 40 + crates/libs/core/src/types.rs | 173 + crates/runtimes/cf-workers/Cargo.toml | 38 + crates/runtimes/cf-workers/README.md | 143 + .../.cache/wrangler/wrangler-account.json | 6 + .../cf-workers/node_modules/.mf/cf.json | 1 + crates/runtimes/cf-workers/src/body.rs | 122 + crates/runtimes/cf-workers/src/client.rs | 199 + crates/runtimes/cf-workers/src/lib.rs | 232 ++ crates/runtimes/cf-workers/src/source_api.rs | 227 ++ .../cf-workers/src/source_resolver.rs | 390 ++ .../runtimes/cf-workers/src/tracing_layer.rs | 117 + crates/runtimes/cf-workers/wrangler.toml | 80 + crates/runtimes/server/Cargo.toml | 22 + crates/runtimes/server/README.md | 115 + crates/runtimes/server/src/bin/s3-proxy.rs | 54 + crates/runtimes/server/src/body.rs | 63 + crates/runtimes/server/src/client.rs | 93 + crates/runtimes/server/src/lib.rs | 12 + crates/runtimes/server/src/server.rs | 141 + deploy/.gitignore | 8 - deploy/.npmignore | 6 - deploy/.nvmrc | 1 - deploy/README.md | 5 - deploy/bin/deploy.ts | 33 - deploy/cdk.json | 99 - deploy/lib/data-proxy-stack.ts | 38 - deploy/lib/source-data-proxy.ts | 127 - deploy/lib/vercel-api-proxy.ts | 113 - deploy/package-lock.json | 678 ---- deploy/package.json | 22 - deploy/tsconfig.json | 31 - docker-compose.yml | 39 + scripts/build-push.sh | 4 - scripts/deploy.sh | 47 - scripts/run.sh | 3 - scripts/tag-release.sh | 29 - scripts/task_definition.json | 57 - src/apis/mod.rs | 31 - src/apis/source/mod.rs | 691 ---- src/apis/source/types.rs | 437 --- src/backends/azure.rs | 305 -- src/backends/common.rs | 170 - src/backends/mod.rs | 3 - src/backends/s3.rs | 382 -- src/main.rs | 502 --- src/utils/api.rs | 45 - src/utils/auth.rs | 505 --- src/utils/core.rs | 99 - src/utils/errors.rs | 654 ---- src/utils/mod.rs | 4 - 85 files changed, 8223 insertions(+), 7063 deletions(-) delete mode 100644 CHANGELOG.md create mode 100644 CLAUDE.md delete mode 100644 CODE_OF_CONDUCT.md delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE create mode 100644 config.example.toml create mode 100644 crates/libs/auth/Cargo.toml create mode 100644 crates/libs/auth/README.md create mode 100644 crates/libs/auth/src/jwks.rs create mode 100644 crates/libs/auth/src/lib.rs create mode 100644 crates/libs/auth/src/sts.rs create mode 100644 crates/libs/core/Cargo.toml create mode 100644 crates/libs/core/README.md create mode 100644 crates/libs/core/src/auth.rs create mode 100644 crates/libs/core/src/backend.rs create mode 100644 crates/libs/core/src/config/cached.rs create mode 100644 crates/libs/core/src/config/dynamodb.rs create mode 100644 crates/libs/core/src/config/http.rs create mode 100644 crates/libs/core/src/config/mod.rs create mode 100644 crates/libs/core/src/config/postgres.rs create mode 100644 crates/libs/core/src/config/static_file.rs create mode 100644 crates/libs/core/src/error.rs create mode 100644 crates/libs/core/src/lib.rs create mode 100644 crates/libs/core/src/maybe_send.rs create mode 100644 crates/libs/core/src/proxy.rs create mode 100644 crates/libs/core/src/resolver.rs create mode 100644 crates/libs/core/src/s3/list_rewrite.rs create mode 100644 crates/libs/core/src/s3/mod.rs create mode 100644 crates/libs/core/src/s3/request.rs create mode 100644 crates/libs/core/src/s3/response.rs create mode 100644 crates/libs/core/src/stream.rs create mode 100644 crates/libs/core/src/types.rs create mode 100644 crates/runtimes/cf-workers/Cargo.toml create mode 100644 crates/runtimes/cf-workers/README.md create mode 100644 crates/runtimes/cf-workers/node_modules/.cache/wrangler/wrangler-account.json create mode 100644 crates/runtimes/cf-workers/node_modules/.mf/cf.json create mode 100644 crates/runtimes/cf-workers/src/body.rs create mode 100644 crates/runtimes/cf-workers/src/client.rs create mode 100644 crates/runtimes/cf-workers/src/lib.rs create mode 100644 crates/runtimes/cf-workers/src/source_api.rs create mode 100644 crates/runtimes/cf-workers/src/source_resolver.rs create mode 100644 crates/runtimes/cf-workers/src/tracing_layer.rs create mode 100644 crates/runtimes/cf-workers/wrangler.toml create mode 100644 crates/runtimes/server/Cargo.toml create mode 100644 crates/runtimes/server/README.md create mode 100644 crates/runtimes/server/src/bin/s3-proxy.rs create mode 100644 crates/runtimes/server/src/body.rs create mode 100644 crates/runtimes/server/src/client.rs create mode 100644 crates/runtimes/server/src/lib.rs create mode 100644 crates/runtimes/server/src/server.rs delete mode 100644 deploy/.gitignore delete mode 100644 deploy/.npmignore delete mode 100644 deploy/.nvmrc delete mode 100644 deploy/README.md delete mode 100644 deploy/bin/deploy.ts delete mode 100644 deploy/cdk.json delete mode 100644 deploy/lib/data-proxy-stack.ts delete mode 100644 deploy/lib/source-data-proxy.ts delete mode 100644 deploy/lib/vercel-api-proxy.ts delete mode 100644 deploy/package-lock.json delete mode 100644 deploy/package.json delete mode 100644 deploy/tsconfig.json create mode 100644 docker-compose.yml delete mode 100755 scripts/build-push.sh delete mode 100755 scripts/deploy.sh delete mode 100755 scripts/run.sh delete mode 100755 scripts/tag-release.sh delete mode 100644 scripts/task_definition.json delete mode 100644 src/apis/mod.rs delete mode 100644 src/apis/source/mod.rs delete mode 100644 src/apis/source/types.rs delete mode 100644 src/backends/azure.rs delete mode 100644 src/backends/common.rs delete mode 100644 src/backends/mod.rs delete mode 100644 src/backends/s3.rs delete mode 100644 src/main.rs delete mode 100644 src/utils/api.rs delete mode 100644 src/utils/auth.rs delete mode 100644 src/utils/core.rs delete mode 100644 src/utils/errors.rs delete mode 100644 src/utils/mod.rs diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 528f75d..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,166 +0,0 @@ -# Changelog - -## [1.0.3](https://github.com/source-cooperative/data.source.coop/compare/v1.0.2...v1.0.3) (2025-10-24) - - -### Performance Improvements - -* persist source client across requests ([#95](https://github.com/source-cooperative/data.source.coop/issues/95)) ([453b2a8](https://github.com/source-cooperative/data.source.coop/commit/453b2a80ff22fc6953f879019aafa77163b4b2b8)) - -## [1.0.2](https://github.com/source-cooperative/data.source.coop/compare/v1.0.1...v1.0.2) (2025-10-09) - - -### Bug Fixes - -* expose content-range header ([#93](https://github.com/source-cooperative/data.source.coop/issues/93)) ([f3fbc3d](https://github.com/source-cooperative/data.source.coop/commit/f3fbc3d650140d28a2c876c9e63fab6004c1c68b)) - -## [1.0.1](https://github.com/source-cooperative/data.source.coop/compare/v1.0.0...v1.0.1) (2025-09-30) - - -### Bug Fixes - -* update source api types to match S2 codebase ([#90](https://github.com/source-cooperative/data.source.coop/issues/90)) ([6dec35d](https://github.com/source-cooperative/data.source.coop/commit/6dec35d60563943fac1d40ce48139859631e540f)) - -## [1.0.0](https://github.com/source-cooperative/data.source.coop/compare/v0.1.29...v1.0.0) (2025-08-21) - - -### ⚠ BREAKING CHANGES - -* update to accomodate Product in s2 API - -### Features - -* add headers to requests to source API ([#81](https://github.com/source-cooperative/data.source.coop/issues/81)) ([edda62f](https://github.com/source-cooperative/data.source.coop/commit/edda62f37d7914cb76209fe7b7209a78e1e49b3c)) -* update to accomodate Product in s2 API ([be44f43](https://github.com/source-cooperative/data.source.coop/commit/be44f43e5497fbb26d46d180162cfffb269dce1b)) -* use Squid proxy for communication with Vercel API ([#85](https://github.com/source-cooperative/data.source.coop/issues/85)) ([25438c3](https://github.com/source-cooperative/data.source.coop/commit/25438c362e9cd1c7d52f5c4d2932542466eef01a)) - - -### Bug Fixes - -* don't specify accept-encoding, letting reqwest handle decompression automatically ([c914e77](https://github.com/source-cooperative/data.source.coop/commit/c914e77b8d0495ec229a2575f8aebbeb41947e8d)) -* lowercase header names ([6b141fc](https://github.com/source-cooperative/data.source.coop/commit/6b141fc34da307593d4fa2526b19becf2f4e1a12)) -* **model:** mv tags & roles to metadata ([571eb94](https://github.com/source-cooperative/data.source.coop/commit/571eb9476cc439b04cf0ecd959e8ca83b3107ccc)) -* update data model to match API ([b6d4032](https://github.com/source-cooperative/data.source.coop/commit/b6d40327a2eda5ff8fb5f69bc7d9e7b67a5d04e2)) -* update source api emails struct ([fbdd02a](https://github.com/source-cooperative/data.source.coop/commit/fbdd02a31627a0f3deb68dc7acd613b041c00bdb)) - - -### Miscellaneous Chores - -* fix release version ([a7cbe0f](https://github.com/source-cooperative/data.source.coop/commit/a7cbe0fbe222beca84db6d2e9ed98da2c9cda42c)) -* fix release version ([e97c41f](https://github.com/source-cooperative/data.source.coop/commit/e97c41fff7b9f5cd5645c97da4ed2c7d2d143d65)) - -## [0.1.29](https://github.com/source-cooperative/data.source.coop/compare/v0.1.28...v0.1.29) (2025-05-29) - - -### Bug Fixes - -* **errors:** pass through client error status codes ([#77](https://github.com/source-cooperative/data.source.coop/issues/77)) ([fe383dd](https://github.com/source-cooperative/data.source.coop/commit/fe383dd08f95d2b6109efa34521815990ece9e0b)) -* **logging:** only log server errors, remove unnecessary details from logs ([#75](https://github.com/source-cooperative/data.source.coop/issues/75)) ([496373c](https://github.com/source-cooperative/data.source.coop/commit/496373c70e77f22f064182641c37ac0f1c6fbef7)) - -## [0.1.28](https://github.com/source-cooperative/data.source.coop/compare/v0.1.27...v0.1.28) (2025-05-28) - - -### Bug Fixes - -* handle unknown Rusoto errors with 404 status ([#73](https://github.com/source-cooperative/data.source.coop/issues/73)) ([8375d50](https://github.com/source-cooperative/data.source.coop/commit/8375d5013a8e559cb2c365722859c70c709ebc68)) - -## [0.1.27](https://github.com/source-cooperative/data.source.coop/compare/v0.1.26...v0.1.27) (2025-05-27) - - -### Bug Fixes - -* treat missing objects as 404s ([#69](https://github.com/source-cooperative/data.source.coop/issues/69)) ([8f4efbf](https://github.com/source-cooperative/data.source.coop/commit/8f4efbf897afaa354b5aab4d5393d69939249ab1)) - -## [0.1.26](https://github.com/source-cooperative/data.source.coop/compare/v0.1.25...v0.1.26) (2025-05-15) - - -### Bug Fixes - -* Rectify crate version ([#61](https://github.com/source-cooperative/data.source.coop/issues/61)) ([00fa9db](https://github.com/source-cooperative/data.source.coop/commit/00fa9db0cc6dee84d7abbfcf9d633a41d1a24f2d)) - - -### Refactor - -* refactor: simplify repository fetching logic in SourceAPI ([#62](https://github.com/source-cooperative/data.source.coop/issues/62)) ([c739a3a](https://github.com/source-cooperative/data.source.coop/commit/c739a3ad2501ac5c8e0bf9a8f6ccf4c8632b7e61)) - -## [0.1.25](https://github.com/source-cooperative/data.source.coop/compare/v0.1.24...v0.1.25) (2025-05-13) - - -### Improvements - -* Add more clarity to cloud provider errors ([#60](https://github.com/source-cooperative/data.source.coop/pull/60)) ([29837a3](https://github.com/source-cooperative/data.source.coop/commit/29837a357172161037a33ab0dad32c0ae3744007)) - - -## [0.1.24](https://github.com/source-cooperative/data.source.coop/compare/v0.1.23...v0.1.24) (2025-05-15) - - -### Improvements - -* Observability/convert error handling ([#59](https://github.com/source-cooperative/data.source.coop/pull/59)) ([562c2de](https://github.com/source-cooperative/data.source.coop/commit/562c2dea3b50c643b749d50a7419fdad991e9cd4)) - -## [0.1.23](https://github.com/source-cooperative/data.source.coop/compare/v0.1.22...v0.1.23) (2025-05-15) - - -### Improvements - -* Log unexpected errors ([d339a01](https://github.com/source-cooperative/data.source.coop/commit/d339a01a43ce2fe01745dffa17e410ed5a156ec4)) - -## [0.1.22](https://github.com/source-cooperative/data.source.coop/compare/v0.1.21...v0.1.22) (2025-05-15) - - -### Bug Fixes - -* File empty on mv ([#54](https://github.com/source-cooperative/data.source.coop/pull/54)) ([d4e329e](https://github.com/source-cooperative/data.source.coop/commit/d4e329e5424cd66ad7930a90685388385e684147)) - - -### Improvements - -* Updated release versions ([#56](https://github.com/source-cooperative/data.source.coop/pull/56)) ([c8b44b6](https://github.com/source-cooperative/data.source.coop/commit/c8b44b68b9b672beebc20324e2c63d34675ad48d)) -* More targetted error handling ([#58](https://github.com/source-cooperative/data.source.coop/pull/58)) ([90e3475](https://github.com/source-cooperative/data.source.coop/commit/90e34750ceabe7281e3cc5dfb003982240e83217)) - -## [0.1.21](https://github.com/source-cooperative/data.source.coop/compare/v0.1.20...v0.1.21) (2025-03-11) - - -### Bug Fixes - -* file empty on mv ([#51](https://github.com/source-cooperative/data.source.coop/issues/51)) ([1f1b3fa](https://github.com/source-cooperative/data.source.coop/commit/1f1b3fa24b175162965281a50c4f50592e1046f8)) - -## [0.1.20](https://github.com/source-cooperative/data.source.coop/compare/v0.1.19...v0.1.20) (2024-12-03) - - -### Bug Fixes - -* Fixed the slow response of the ListObjects call. ([#32](https://github.com/source-cooperative/data.source.coop/issues/32)) ([6afcf13](https://github.com/source-cooperative/data.source.coop/commit/6afcf13ec15b9cc79f5d6a2aef55b3d269a14e16)) - -## [0.1.19](https://github.com/source-cooperative/data.source.coop/compare/v0.1.18...v0.1.19) (2024-11-28) - - -### Bug Fixes - -* Fixed issues in listing bucket at account level. ([#28](https://github.com/source-cooperative/data.source.coop/issues/28)) ([073d2ea](https://github.com/source-cooperative/data.source.coop/commit/073d2ea34fb5f4c00716605538c585a0a486588a)) - -## [0.1.18](https://github.com/source-cooperative/data.source.coop/compare/v0.1.17...v0.1.18) (2024-11-22) - - -### Bug Fixes - -* check for empty access key id. ([#24](https://github.com/source-cooperative/data.source.coop/issues/24)) ([8df8242](https://github.com/source-cooperative/data.source.coop/commit/8df8242f1772705d672cf7594427333fc68627cb)) - -## [0.1.17](https://github.com/source-cooperative/data.source.coop/compare/v0.1.16...v0.1.17) (2024-11-13) - - -### Bug Fixes - -* Fixed the issue in request authorization. Decoded the request path before its encoded again. ([#20](https://github.com/source-cooperative/data.source.coop/issues/20)) ([dc9eb84](https://github.com/source-cooperative/data.source.coop/commit/dc9eb84009eead0dbecd0990886f69811ca93abd)) - -Version 0.1.16 -------------- -* Handled the boto3 download object with range request pattern `start-` which is a valid request to fetch the bytes from start till the total bytes. - -Version 0.1.15 --------------- -* Added `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `CHANGELOG.rst`, Github issue templates, and Github pull request template. - -Version 0.1.14 --------------- -* Released initial open-source version of the project. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..bdc8f2c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,33 @@ +# S3 Proxy Gateway + +Multi-runtime S3 gateway proxy in Rust. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. + +## Workspace Structure + +- `crates/libs/core` — Core proxy logic, traits, config, S3 request parsing +- `crates/libs/auth` — Authentication (SigV4 verification, JWT) +- `crates/runtimes/server` — Tokio/Hyper server runtime +- `crates/runtimes/cf-workers` — Cloudflare Workers runtime (WASM) + +## Build Commands + +```bash +# Check/build default workspace members (excludes cf-workers) +cargo check +cargo build + +# CF Workers crate MUST be checked/built with the wasm32 target: +cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown + +# Run tests +cargo test +``` + +## Key Architecture Notes + +- **RequestResolver pattern**: `ProxyHandler` is generic over a `RequestResolver` trait. The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.handle_request()`. +- **MaybeSend pattern**: Core traits use `MaybeSend`/`MaybeSync` (defined in `crates/libs/core/src/maybe_send.rs`) instead of `Send`/`Sync`. On native targets these resolve to `Send`/`Sync`; on `wasm32` they are no-op blanket traits. This allows the CF Workers runtime to use `!Send` JS interop types (`JsValue`, `ReadableStream`, etc.). +- **cf-workers is excluded from `default-members`** in the root `Cargo.toml` because WASM types are `!Send` and will fail to compile on native targets. Always use `--target wasm32-unknown-unknown` when working with this crate. +- **Streaming passthrough**: The CF Workers runtime passes `ReadableStream` bodies through opaquely — bytes never enter Rust memory for GET/PUT requests. The `WorkerBody` enum wraps `Bytes`, `ReadableStream`, or `Empty`. +- **Config loading** (CF Workers): `PROXY_CONFIG` can be either a JSON string (via `wrangler secret`) or a JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`). Both formats are handled. +- **List response rewriting**: When a resolver returns `ResolvedAction::Proxy` with a `ListRewrite`, the proxy handler buffers the (small) list XML response and rewrites `` and `` element values — stripping a backend prefix and optionally prepending a new one. This is handled in `crates/libs/core/src/s3/list_rewrite.rs`. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 5eac9d6..0000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,132 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, caste, color, religion, or sexual -identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall - community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or advances of - any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email address, - without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -[stac-spec-admins@googlegroups.com](mailto:stac-spec-admins@googlegroups.com). -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of -actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or permanent -ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the -community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.1, available at -[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. - -Community Impact Guidelines were inspired by -[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. - -For answers to common questions about this code of conduct, see the FAQ at -[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at -[https://www.contributor-covenant.org/translations][translations]. - -[homepage]: https://www.contributor-covenant.org -[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html -[Mozilla CoC]: https://github.com/mozilla/diversity -[FAQ]: https://www.contributor-covenant.org/faq -[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 9993ccd..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,90 +0,0 @@ -# How to contribute to Source Cooperative - -## Bugs - -- **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/source-cooperative/data.source.coop/issues). - -- If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/source-cooperative/data.source.coop/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. - -- If possible, use the relevant bug report templates to create the issue. - -#### Did you write a patch that fixes a bug? - -- Open a new GitHub pull request with the patch. - -- Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. - -## Features - -Prior to implementing a feature, it is recommended to [create an issue](https://github.com/source-cooperative/data.source.coop/issues/new) on GitHub and describe the new feature or change you would like to add. - -## General Communication - -Ask any question about how to use Source Cooperative in the [source-cooperative slack channel](https://join.slack.com/t/sourcecoop/shared_invite/zt-212sakf1j-fONCD4lZ_v2HP2PDpTr2dw). - -## Contributing Code - -To make contributions to this codebase, please create a pull request of a feature branch to the `main` branch. The PR title should conform to [Conventional Commits](http://conventionalcommits.org/en/v1.0.0/). - -> [!TIP] -> The `CHANGELOG.md` and the project version within `Cargo.toml` are managed automatically within our CICD pipeline. There is typically no need for individual developers to alter these values. - -### Releases - -Releases are automated via the [Release Please action](https://github.com/googleapis/release-please-action/). As contributions are made to `main`, a release PR will be kept up-to-date to represent the upcoming release. When that PR is merged, a new Github Release will be generated. - -### Deployments - -Merges to the `main` branch trigger deployment to the development instance of the proxy. - -New releases trigger deployment to the production instance of the proxy. - -
- -Manual Deployment Steps - -**⚠️ Manual deployment should only be necessary in extreme circumstances. Automated deployments via GitHub Workflows are preferred. ⚠️** - -## Deployment - -Before you begin the deployment process, ensure that you have the `SOURCE_KEY` environment variable set with the production key. - -### Tagging Release - -After committing your changes, tag the release and bump the version with the following command: - -``` -./scripts/tag-release.sh -``` - -### Building and Pushing Image - -To build and push the docker image to ECR, run the following command: - -``` -./scripts/build-push.sh -``` - -### Deploying to ECS - -To deploy the image to ECS, run the following command: - -``` -./scripts/deploy.sh -``` - -### Rolling Back a Deployment - -To roll back a deployment, first checkout the code for the version that you want to roll back to. For example: - -``` -git checkout v0.1.12 -``` - -Next, deploy the version to ECS: - -``` -./scripts/deploy.sh -``` - -
diff --git a/Cargo.lock b/Cargo.lock index 52f0d47..0bad143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,445 +3,334 @@ version = 4 [[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - -[[package]] -name = "actix-codec" -version = "0.5.2" +name = "aho-corasick" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ - "bitflags 2.6.0", - "bytes", - "futures-core", - "futures-sink", "memchr", - "pin-project-lite", - "tokio", - "tokio-util", - "tracing", ] [[package]] -name = "actix-cors" -version = "0.7.0" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e772b3bcafe335042b5db010ab7c09013dad6eac4915c91d8d50902769f331" -dependencies = [ - "actix-utils", - "actix-web", - "derive_more", - "futures-util", - "log", - "once_cell", - "smallvec", -] +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] -name = "actix-http" -version = "3.9.0" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-tls", - "actix-utils", - "ahash", - "base64 0.22.1", - "bitflags 2.6.0", - "bytes", - "bytestring", - "derive_more", - "encoding_rs", - "futures-core", - "h2", - "http 0.2.12", - "httparse", - "httpdate", - "itoa", - "language-tags", - "local-channel", - "mime", - "percent-encoding", - "pin-project-lite", - "rand 0.8.5", - "sha1", - "smallvec", - "tokio", - "tokio-util", - "tracing", + "libc", ] [[package]] -name = "actix-macros" -version = "0.2.4" +name = "anyhow" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ + "proc-macro2", "quote", "syn", ] [[package]] -name = "actix-router" -version = "0.5.3" +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ - "bytestring", - "cfg-if", - "http 0.2.12", - "regex-lite", - "serde", - "tracing", + "num-traits", ] [[package]] -name = "actix-rt" -version = "2.10.0" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" -dependencies = [ - "futures-core", - "tokio", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "actix-server" -version = "2.5.0" +name = "autocfg" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" -dependencies = [ - "actix-rt", - "actix-service", - "actix-utils", - "futures-core", - "futures-util", - "mio", - "socket2", - "tokio", - "tracing", -] +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "actix-service" -version = "2.0.2" +name = "aws-credential-types" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +checksum = "e26bbf46abc608f2dc61fd6cb3b7b0665497cc259a21520151ed98f8b37d2c79" dependencies = [ - "futures-core", - "paste", - "pin-project-lite", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", ] [[package]] -name = "actix-tls" -version = "3.4.0" +name = "aws-lc-rs" +version = "1.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac453898d866cdbecdbc2334fe1738c747b4eba14a677261f2b768ba05329389" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ - "actix-rt", - "actix-service", - "actix-utils", - "futures-core", - "impl-more", - "pin-project-lite", - "tokio", - "tokio-rustls 0.23.4", - "tokio-util", - "tracing", - "webpki-roots", + "aws-lc-sys", + "zeroize", ] [[package]] -name = "actix-utils" -version = "3.0.1" +name = "aws-lc-sys" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" dependencies = [ - "local-waker", - "pin-project-lite", + "cc", + "cmake", + "dunce", + "fs_extra", ] [[package]] -name = "actix-web" -version = "4.9.0" +name = "aws-runtime" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" +checksum = "b0f92058d22a46adf53ec57a6a96f34447daf02bff52e8fb956c66bcd5c6ac12" dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", - "actix-router", - "actix-rt", - "actix-server", - "actix-service", - "actix-tls", - "actix-utils", - "actix-web-codegen", - "ahash", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", - "bytestring", - "cfg-if", - "derive_more", - "encoding_rs", - "futures-core", - "futures-util", - "impl-more", - "itoa", - "language-tags", - "log", - "mime", - "once_cell", + "bytes-utils", + "fastrand", + "http 1.4.0", + "http-body 1.0.1", + "percent-encoding", "pin-project-lite", - "regex-lite", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "socket2", - "time", - "url", -] - -[[package]] -name = "actix-web-codegen" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" -dependencies = [ - "actix-router", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "addr2line" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" -dependencies = [ - "gimli", + "tracing", + "uuid", ] [[package]] -name = "adler" -version = "1.0.2" +name = "aws-sdk-dynamodb" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "82d2214c2ad3a175d3ece5a5af26916c29caa3e12e9e05b3cb8ed5e837b54b67" dependencies = [ - "cfg-if", - "getrandom 0.2.15", - "once_cell", - "version_check", - "zerocopy", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", ] [[package]] -name = "aho-corasick" -version = "1.1.3" +name = "aws-sigv4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "68f6ae9b71597dc5fd115d52849d7a5556ad9265885ad3492ea8d73b93bbc46e" dependencies = [ - "memchr", + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2", + "time", + "tracing", ] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "aws-smithy-async" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "3cba48474f1d6807384d06fec085b909f5807e16653c5af5c45dfe89539f0b70" dependencies = [ - "libc", + "futures-util", + "pin-project-lite", + "tokio", ] [[package]] -name = "anyhow" -version = "1.0.86" +name = "aws-smithy-http" +version = "0.63.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +checksum = "af4a8a5fe3e4ac7ee871237c340bbce13e982d37543b65700f4419e039f5d78e" dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", "futures-core", -] - -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener 5.3.1", - "event-listener-strategy", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", "pin-project-lite", + "pin-utils", + "tracing", ] [[package]] -name = "async-trait" -version = "0.1.81" +name = "aws-smithy-http-client" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "0709f0083aa19b704132684bc26d3c868e06bd428ccc4373b0b55c3e8748a58b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", ] [[package]] -name = "atty" -version = "0.2.14" +name = "aws-smithy-json" +version = "0.62.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "aws-smithy-types", ] [[package]] -name = "autocfg" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "azure_core" -version = "0.20.0" +name = "aws-smithy-observability" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ce3de4b65b1ee2667c81d1fc692949049502a4cf9c38118d811d6d79a7eaef" +checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.15", - "hmac 0.12.1", - "http-types", - "once_cell", - "paste", - "pin-project", - "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.12.5", - "rustc_version", - "serde", - "serde_json", - "sha2 0.10.8", - "time", - "tracing", - "url", - "uuid", + "aws-smithy-runtime-api", ] [[package]] -name = "azure_storage" -version = "0.20.0" +name = "aws-smithy-runtime" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9713002fc30956a9f4061cdbc2e912ff739c6160e138ad3b6d992b3bcedccc6d" +checksum = "8fd3dfc18c1ce097cf81fced7192731e63809829c6cbf933c1ec47452d08e1aa" dependencies = [ - "RustyXML", - "async-lock", - "async-trait", - "azure_core", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", - "serde", - "serde_derive", - "time", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", "tracing", - "url", - "uuid", ] [[package]] -name = "azure_storage_blobs" -version = "0.20.0" +name = "aws-smithy-runtime-api" +version = "1.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5b3a31dd8f920739437b827d0c9f9a4011eb3f06f79a121764aa11af6c51ee2" +checksum = "8c55e0837e9b8526f49e0b9bfa9ee18ddee70e853f5bc09c5d11ebceddcb0fec" dependencies = [ - "RustyXML", - "azure_core", - "azure_storage", - "azure_svc_blobstorage", + "aws-smithy-async", + "aws-smithy-types", "bytes", - "futures", - "serde", - "serde_derive", - "serde_json", - "time", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", "tracing", - "url", - "uuid", + "zeroize", ] [[package]] -name = "azure_svc_blobstorage" -version = "0.20.0" +name = "aws-smithy-types" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef37ba6180df451042f1c277d4d0898e2447f0a5d5072e0ff11ee6ea5e7ef38" +checksum = "576b0d6991c9c32bc14fc340582ef148311f924d41815f641a308b5d11e8e7cd" dependencies = [ - "azure_core", + "base64-simd", "bytes", - "futures", - "log", - "once_cell", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", "serde", - "serde_json", "time", + "tokio", + "tokio-util", ] [[package]] -name = "backtrace" -version = "0.3.73" +name = "aws-types" +version = "1.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "6c50f3cdf47caa8d01f2be4a6663ea02418e892f9bbfd82c7b9a3a37eaccdd3a" dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -449,24 +338,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "bitflags" -version = "1.3.2" +name = "base64-simd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] [[package]] -name = "bitflags" -version = "2.6.0" +name = "base64ct" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] -name = "block-buffer" -version = "0.9.0" +name = "bitflags" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" dependencies = [ - "generic-array", + "serde_core", ] [[package]] @@ -480,9 +373,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "byteorder" @@ -492,74 +385,91 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] -name = "bytestring" -version = "1.3.1" +name = "bytes-utils" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", + "either", ] [[package]] name = "cc" -version = "1.1.10" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] -name = "common-s3-headers" -version = "1.0.0" +name = "cmake" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f177814b579c39ae2325720be922d21fab28ff22fe81aa27c79625326ce19db" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" dependencies = [ - "hex", - "hmac 0.12.1", - "percent-encoding", - "sha2 0.10.8", - "time", - "url", + "cc", ] [[package]] name = "concurrent-queue" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "crossbeam-utils", + "cfg-if", + "wasm-bindgen", ] [[package]] -name = "convert_case" -version = "0.4.0" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "core-foundation" @@ -571,6 +481,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -579,105 +499,71 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] -name = "crc32fast" -version = "1.4.2" +name = "crc" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ - "cfg-if", + "crc-catalog", ] [[package]] -name = "crossbeam-channel" -version = "0.5.13" +name = "crc-catalog" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" -dependencies = [ - "crossbeam-utils", -] +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] -name = "crossbeam-epoch" -version = "0.9.18" +name = "crossbeam-queue" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", ] [[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ct-logs" -version = "0.8.0" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "sct 0.6.1", + "const-oid", + "pem-rfc7468", + "zeroize", ] [[package]] name = "deranged" -version = "0.3.11" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "cc3dc5ad92c2e2d1c193bbbbdf2ea477cb81331de4f3103f267ca18368b988c4" dependencies = [ "powerfmt", - "serde", -] - -[[package]] -name = "derive_more" -version = "0.99.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", ] [[package]] @@ -686,87 +572,76 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", + "const-oid", "crypto-common", "subtle", ] [[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "libc", - "redox_users", - "winapi", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "dyn-clone" -version = "1.0.17" +name = "dotenvy" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] -name = "encoding_rs" -version = "0.8.34" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "env_logger" -version = "0.9.3" +name = "either" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", + "serde", ] [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] -name = "event-listener" -version = "2.5.3" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ "concurrent-queue", "parking", @@ -774,29 +649,27 @@ dependencies = [ ] [[package]] -name = "event-listener-strategy" -version = "0.5.2" +name = "fastrand" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" -dependencies = [ - "event-listener 5.3.1", - "pin-project-lite", -] +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "fastrand" -version = "1.9.0" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "fastrand" -version = "2.1.0" +name = "flume" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] [[package]] name = "fnv" @@ -805,49 +678,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] [[package]] -name = "futures" -version = "0.3.30" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -855,15 +710,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -871,31 +726,27 @@ dependencies = [ ] [[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] -name = "futures-lite" -version = "1.13.0" +name = "futures-intrusive" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ - "fastrand 1.9.0", "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", + "lock_api", + "parking_lot", ] +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -904,23 +755,22 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -944,39 +794,49 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "wasi", + "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.2.15" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "r-efi", + "wasip2", "wasm-bindgen", ] [[package]] -name = "gimli" -version = "0.29.0" +name = "getrandom" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] [[package]] name = "h2" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" dependencies = [ "bytes", "fnv", @@ -991,26 +851,56 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "hashlink" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "libc", + "hashbrown 0.15.5", ] [[package]] -name = "hermit-abi" -version = "0.3.9" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hex" @@ -1019,13 +909,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hmac" -version = "0.11.0" +name = "hkdf" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac", ] [[package]] @@ -1034,7 +923,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", ] [[package]] @@ -1050,12 +948,11 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -1077,47 +974,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.4.0", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.1.0", + "futures-core", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel", - "base64 0.13.1", - "futures-lite", - "infer", - "pin-project-lite", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url", -] - [[package]] name = "httparse" -version = "1.9.4" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -1125,30 +1002,24 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1157,18 +1028,22 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", - "http 1.1.0", + "futures-core", + "h2 0.4.13", + "http 1.4.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -1176,80 +1051,74 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.22.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "ct-logs", "futures-util", - "hyper 0.14.30", + "http 0.2.12", + "hyper 0.14.32", "log", - "rustls 0.19.1", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.22.0", - "webpki 0.21.4", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper 0.14.30", - "native-tls", + "rustls 0.21.12", "tokio", - "tokio-native-tls", + "tokio-rustls 0.24.1", ] [[package]] -name = "hyper-tls" -version = "0.6.0" +name = "hyper-rustls" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "bytes", - "http-body-util", - "hyper 1.4.1", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", - "native-tls", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", "tokio", - "tokio-native-tls", + "tokio-rustls 0.26.4", "tower-service", + "webpki-roots 1.0.6", ] [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.8.1", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", + "system-configuration", "tokio", - "tower", + "tower-layer", "tower-service", "tracing", + "windows-registry", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", "windows-core", ] @@ -1263,312 +1132,375 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" -version = "0.5.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "impl-more" -version = "0.1.6" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] [[package]] name = "indexmap" -version = "2.3.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] -name = "infer" -version = "0.2.3" +name = "ipnet" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] -name = "instant" -version = "0.1.13" +name = "iri-string" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ - "cfg-if", + "memchr", + "serde", ] [[package]] -name = "ipnet" -version = "2.9.0" +name = "itoa" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] -name = "itoa" -version = "1.0.11" +name = "jobserver" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ + "once_cell", "wasm-bindgen", ] -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" + +[[package]] +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.6.0", + "bitflags", "libc", + "redox_syscall 0.7.1", ] [[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - -[[package]] -name = "local-channel" -version = "0.1.5" +name = "libsqlite3-sys" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ - "futures-core", - "futures-sink", - "local-waker", + "pkg-config", + "vcpkg", ] [[package]] -name = "local-waker" -version = "0.1.4" +name = "litemap" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.22" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "md-5" -version = "0.9.1" +name = "lru-slab" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "memchr" -version = "2.7.4" +name = "matchers" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] [[package]] -name = "mime" -version = "0.3.17" +name = "matchit" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] -name = "miniz_oxide" -version = "0.7.4" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "adler", + "cfg-if", + "digest", ] +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + [[package]] name = "mio" -version = "1.0.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ - "hermit-abi 0.3.9", "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi", + "windows-sys 0.61.2", ] [[package]] -name = "moka" -version = "0.12.8" +name = "nu-ansi-term" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "async-lock", - "async-trait", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "event-listener 5.3.1", - "futures-util", - "once_cell", - "parking_lot", - "quanta", - "rustc_version", - "smallvec", - "tagptr", - "thiserror 1.0.63", - "triomphe", - "uuid", + "windows-sys 0.61.2", ] [[package]] -name = "native-tls" -version = "0.2.12" +name = "num-bigint-dig" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", ] [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] -name = "num-traits" -version = "0.2.19" +name = "num-integer" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", + "num-traits", ] [[package]] -name = "object" -version = "0.36.3" +name = "num-iter" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ - "memchr", + "autocfg", + "num-integer", + "num-traits", ] [[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - -[[package]] -name = "openssl" -version = "0.10.66" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "bitflags 2.6.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "autocfg", + "libm", ] [[package]] -name = "openssl-macros" -version = "0.1.1" +name = "once_cell" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] -name = "openssl-sys" -version = "0.9.103" +name = "outref" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -1576,43 +1508,46 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", - "windows-targets 0.52.6", + "windows-link", ] [[package]] -name = "paste" -version = "1.0.15" +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", @@ -1621,9 +1556,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -1631,11 +1566,41 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] [[package]] name = "powerfmt" @@ -1645,79 +1610,112 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ "zerocopy", ] [[package]] -name = "proc-macro2" -version = "1.0.95" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ - "unicode-ident", + "proc-macro2", + "syn", ] [[package]] -name = "quanta" -version = "0.12.3" +name = "proc-macro2" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", - "web-sys", - "winapi", + "unicode-ident", ] [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", "serde", ] [[package]] -name = "quick-xml" -version = "0.36.1" +name = "quinn" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ - "memchr", - "serde", + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.36", + "socket2 0.6.2", + "thiserror", + "tokio", + "tracing", + "web-time", ] [[package]] -name = "quote" -version = "1.0.36" +name = "quinn-proto" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ - "proc-macro2", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls 0.23.36", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", ] [[package]] -name = "rand" -version = "0.7.3" +name = "quinn-udp" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ - "getrandom 0.1.16", + "cfg_aliases", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", + "once_cell", + "socket2 0.6.2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1730,13 +1728,13 @@ dependencies = [ ] [[package]] -name = "rand_chacha" -version = "0.2.2" +name = "rand" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1750,12 +1748,13 @@ dependencies = [ ] [[package]] -name = "rand_core" -version = "0.5.1" +name = "rand_chacha" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ - "getrandom 0.1.16", + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1764,64 +1763,41 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom 0.2.17", ] [[package]] -name = "raw-cpuid" -version = "11.2.0" +name = "rand_core" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ - "bitflags 2.6.0", + "getrandom 0.3.4", ] [[package]] name = "redox_syscall" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" -dependencies = [ - "bitflags 2.6.0", -] - -[[package]] -name = "redox_users" -version = "0.4.5" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "getrandom 0.2.15", - "libredox", - "thiserror 1.0.63", + "bitflags", ] [[package]] -name = "regex" -version = "1.10.6" +name = "redox_syscall" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "35985aa610addc02e24fc232012c86fd11f14111180f902b67e2d5331f8ebf2b" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -1830,314 +1806,284 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" - -[[package]] -name = "reqwest" -version = "0.11.27" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", - "hyper-tls 0.5.0", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration", - "tokio", - "tokio-native-tls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg 0.50.0", -] +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-util", - "http 1.1.0", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", - "hyper-tls 0.6.0", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.1.3", + "quinn", + "rustls 0.23.36", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls 0.26.4", "tokio-util", + "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg 0.52.0", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "webpki-roots 1.0.6", ] [[package]] name = "ring" -version = "0.17.8" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.17", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] [[package]] -name = "rusoto_core" -version = "0.47.0" +name = "rsa" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes", - "crc32fast", - "futures", - "http 0.2.12", - "hyper 0.14.30", - "hyper-rustls", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version", - "serde", - "serde_json", - "tokio", - "xml-rs", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", ] [[package]] -name = "rusoto_credential" -version = "0.47.0" +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper 0.14.30", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] -name = "rusoto_s3" -version = "0.47.0" +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048c2fe811a823ad5a9acc976e8bf4f1d910df719dcf44b15c3e96c5b7a51027" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "async-trait", - "bytes", - "futures", - "rusoto_core", - "xml-rs", + "semver", ] [[package]] -name = "rusoto_signature" -version = "0.47.0" +name = "rustls" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ - "base64 0.13.1", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http 0.2.12", - "hyper 0.14.30", "log", - "md-5", - "percent-encoding", - "pin-project-lite", - "rusoto_credential", - "rustc_version", - "serde", - "sha2 0.9.9", - "tokio", + "ring", + "rustls-webpki 0.101.7", + "sct", ] [[package]] -name = "rustc-demangle" -version = "0.1.24" +name = "rustls" +version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.103.9", + "subtle", + "zeroize", +] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "rustls-native-certs" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "semver", + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", ] [[package]] -name = "rustix" -version = "0.38.34" +name = "rustls-pki-types" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ - "bitflags 2.6.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "web-time", + "zeroize", ] [[package]] -name = "rustls" -version = "0.19.1" +name = "rustls-webpki" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "base64 0.13.1", - "log", - "ring 0.16.20", - "sct 0.6.1", - "webpki 0.21.4", + "ring", + "untrusted", ] [[package]] -name = "rustls" -version = "0.20.9" +name = "rustls-webpki" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ - "log", - "ring 0.16.20", - "sct 0.7.1", - "webpki 0.22.4", + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] -name = "rustls-native-certs" -version = "0.5.0" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls 0.19.1", - "schannel", - "security-framework", -] +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] -name = "rustls-pemfile" -version = "1.0.4" +name = "ryu" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "s3-proxy-auth" +version = "0.1.0" dependencies = [ - "base64 0.21.7", + "async-trait", + "base64", + "chrono", + "reqwest", + "rsa", + "s3-proxy-core", + "serde", + "serde_json", + "sha2", + "thiserror", + "tracing", + "uuid", ] [[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +name = "s3-proxy-cf-workers" +version = "0.1.0" dependencies = [ - "base64 0.22.1", - "rustls-pki-types", + "bytes", + "chrono", + "console_error_panic_hook", + "getrandom 0.2.17", + "http 1.4.0", + "js-sys", + "quick-xml", + "s3-proxy-auth", + "s3-proxy-core", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "worker", ] [[package]] -name = "rustls-pki-types" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +name = "s3-proxy-core" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-sdk-dynamodb", + "base64", + "bytes", + "chrono", + "hex", + "hmac", + "http 1.4.0", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "sha2", + "sqlx", + "thiserror", + "tokio", + "toml", + "tracing", + "url", + "uuid", +] [[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +name = "s3-proxy-server" +version = "0.1.0" +dependencies = [ + "bytes", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "reqwest", + "s3-proxy-auth", + "s3-proxy-core", + "serde", + "thiserror", + "tokio", + "toml", + "tracing", + "tracing-subscriber", +] [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2146,34 +2092,24 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - [[package]] name = "sct" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] name = "security-framework" -version = "2.11.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ - "bitflags 2.6.0", - "core-foundation", + "bitflags", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2181,9 +2117,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "321c8673b092a9a42605034a9879d73cb79101ed5fd117bc9a597b89b4e9e61a" dependencies = [ "core-foundation-sys", "libc", @@ -2191,36 +2127,45 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.207" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] -name = "serde-xml-rs" -version = "0.6.0" +name = "serde-wasm-bindgen" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" dependencies = [ - "log", + "js-sys", "serde", - "thiserror 1.0.63", - "xml-rs", + "wasm-bindgen", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.207" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -2229,25 +2174,24 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] -name = "serde_qs" -version = "0.8.5" +name = "serde_spanned" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ - "percent-encoding", "serde", - "thiserror 1.0.63", ] [[package]] @@ -2270,137 +2214,327 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls 0.23.36", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", ] [[package]] -name = "sha2" -version = "0.9.9" +name = "sqlx-macros" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn", ] [[package]] -name = "sha2" -version = "0.10.8" +name = "sqlx-macros-core" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn", + "tokio", + "url", ] [[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.2" +name = "sqlx-mysql" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ - "libc", + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", ] [[package]] -name = "slab" -version = "0.4.9" +name = "sqlx-postgres" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ - "autocfg", + "atoi", + "base64", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", ] [[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "socket2" -version = "0.5.7" +name = "sqlx-sqlite" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "source-data-proxy" -version = "1.0.3" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ - "actix-cors", - "actix-http", - "actix-web", - "async-trait", - "azure_core", - "azure_storage", - "azure_storage_blobs", - "bytes", - "chrono", - "common-s3-headers", - "env_logger", - "futures", + "atoi", + "flume", + "futures-channel", "futures-core", + "futures-executor", + "futures-intrusive", "futures-util", - "hex", - "hmac 0.12.1", + "libsqlite3-sys", "log", - "moka", "percent-encoding", - "pin-project-lite", - "quick-xml 0.36.1", - "reqwest 0.11.27", - "rusoto_core", - "rusoto_credential", - "rusoto_s3", "serde", - "serde-xml-rs", - "serde_json", - "sha2 0.10.8", - "thiserror 2.0.12", - "time", - "tokio", - "tokio-util", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", "url", - "xml-rs", ] [[package]] -name = "spin" -version = "0.5.2" +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "spin" -version = "0.9.8" +name = "stringprep" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.101" +version = "2.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "6e614ed320ac28113fa64972c4262d5dbc89deacdfd00c34a3e4cea073243c12" dependencies = [ "proc-macro2", "quote", @@ -2409,88 +2543,59 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] -name = "sync_wrapper" -version = "1.0.1" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 1.3.2", - "core-foundation", + "bitflags", + "core-foundation 0.9.4", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", ] -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "tempfile" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" -dependencies = [ - "cfg-if", - "fastrand 2.1.0", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "thiserror" -version = "1.0.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" -dependencies = [ - "thiserror-impl 1.0.63", -] - [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -2498,53 +2603,59 @@ dependencies = [ ] [[package]] -name = "thiserror-impl" -version = "2.0.12" +name = "thread_local" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "proc-macro2", - "quote", - "syn", + "cfg-if", ] [[package]] name = "time" -version = "0.3.36" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", - "itoa", - "js-sys", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ "tinyvec_macros", ] @@ -2557,27 +2668,26 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ - "backtrace", "bytes", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", @@ -2585,42 +2695,41 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.1" +name = "tokio-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "native-tls", + "rustls 0.21.12", "tokio", ] [[package]] name = "tokio-rustls" -version = "0.22.0" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.19.1", + "rustls 0.23.36", "tokio", - "webpki 0.21.4", ] [[package]] -name = "tokio-rustls" -version = "0.23.4" +name = "tokio-stream" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ - "rustls 0.20.9", + "futures-core", + "pin-project-lite", "tokio", - "webpki 0.22.4", ] [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -2629,21 +2738,80 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" -version = "0.4.13" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "pin-project", "pin-project-lite", + "sync_wrapper", "tokio", "tower-layer", "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -2658,9 +2826,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -2670,9 +2838,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -2681,18 +2849,42 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ + "log", "once_cell", + "tracing-core", ] [[package]] -name = "triomphe" -version = "0.1.11" +name = "tracing-subscriber" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] [[package]] name = "try-lock" @@ -2702,36 +2894,42 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] [[package]] -name = "untrusted" -version = "0.7.1" +name = "unicode-properties" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -2741,9 +2939,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -2751,16 +2949,29 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" -version = "1.10.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ - "getrandom 0.2.15", - "serde", + "getrandom 0.4.1", + "js-sys", + "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2774,10 +2985,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "waker-fn" -version = "1.2.0" +name = "vsimd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "want" @@ -2790,59 +3001,66 @@ dependencies = [ [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] [[package]] -name = "wasm-bindgen" -version = "0.2.93" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.93" +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2850,28 +3068,53 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -2880,83 +3123,134 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] -name = "webpki" -version = "0.21.4" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "webpki" -version = "0.22.4" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "webpki-roots 1.0.6", ] [[package]] name = "webpki-roots" -version = "0.22.6" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "webpki 0.22.4", + "libredox", + "wasite", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "windows-core" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "windows-implement" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "winapi-util" -version = "0.1.9" +name = "windows-interface" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ - "windows-sys 0.59.0", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows-registry" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-targets 0.52.6", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", ] [[package]] @@ -2979,11 +3273,20 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", ] [[package]] @@ -3010,13 +3313,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3029,6 +3349,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3041,6 +3367,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3053,12 +3385,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3071,6 +3415,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3083,6 +3433,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3095,6 +3451,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3108,54 +3470,277 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winreg" -version = "0.50.0" +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "memchr", ] [[package]] -name = "winreg" -version = "0.52.0" +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "worker" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244647fd7673893058f91f22a0eabd0f45bb50298e995688cb0c4b9837081b19" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "js-sys", + "matchit", + "pin-project", + "serde", + "serde-wasm-bindgen", + "serde_json", + "serde_urlencoded", + "tokio", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "worker-macros", + "worker-sys", +] + +[[package]] +name = "worker-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac7e73ffb164183b57bb67d3efb881681fcd93ef5515ba32a4d022c4a6acc2ce" +dependencies = [ + "async-trait", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-macro-support", + "worker-sys", +] + +[[package]] +name = "worker-sys" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "2a2b96254fcaa9229fd82d886f04be99c4ee8e59c8d80438724aa70039dca838" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] -name = "xml-rs" -version = "0.8.21" +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a77ee7c0de333dcc6da69b177380a0b81e0dacfa4f7344c465a36871ee601" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ - "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", "syn", + "synstructure", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index ea5d2d6..a9ee0c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,54 +1,64 @@ -[package] -name = "source-data-proxy" +[workspace] +members = [ + "crates/libs/core", + "crates/libs/auth", + "crates/runtimes/server", + "crates/runtimes/cf-workers", +] +# Worker crate is excluded from default builds because it contains !Send +# WASM types that only compile correctly for wasm32 targets. Build it +# separately via: cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +default-members = [ + "crates/libs/core", + "crates/libs/auth", + "crates/runtimes/server", +] +resolver = "2" -version = "1.0.3" +[workspace.package] +version = "0.1.0" edition = "2021" +license = "MIT OR Apache-2.0" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] - -quick-xml = { version = "=0.36.1", features = ["serialize"] } -actix-web = { version = "^4", features = [ - "rustls", - "macros", -], default-features = false } -rusoto_core = { version = "0.47", default-features = false, features = [ - "rustls", -] } -rusoto_s3 = { version = "0.47", default-features = false, features = [ - "rustls", -] } -rusoto_credential = { version = "0.47" } -tokio-util = { version = "0.7", features = ["codec"] } -tokio = { version = "1", features = ["full"] } -futures-util = "0.3" -xml-rs = "0.8" -serde = { version = "1.0", features = ["derive"] } -serde-xml-rs = "0.6" -bytes = "1.0" -pin-project-lite = "0.2" -futures = "0.3" -futures-core = "0.3" -log = "0.4" -env_logger = "0.9" +[workspace.dependencies] +# Core +async-trait = "0.1" +thiserror = "2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.8" +bytes = "1" +http = "1" chrono = { version = "0.4", features = ["serde"] } -async-trait = "0.1.81" -azure_storage_blobs = "0.20.0" -azure_storage = "0.20.0" -azure_core = "0.20.0" -time = { version = "0.3", features = ["formatting"] } -url = "2.2.2" -reqwest = { version = "0.11.0", features = ["stream", "json"] } -actix-cors = "0.7.0" -moka = { version = "0.12.8", features = ["future"] } -percent-encoding = "2.1.0" -sha2 = "0.10.6" -hex = "0.4.3" +uuid = { version = "1", features = ["v4", "js"] } +base64 = "0.22" +hex = "0.4" +url = "2" + +# Crypto hmac = "0.12" -actix-http = "^3" -thiserror = "2.0.12" -serde_json = "1.0.141" +sha2 = { version = "0.10", features = ["oid"] } +rsa = "0.9" + +# XML +quick-xml = { version = "0.37", features = ["serialize"] } + +# Auth +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } + +# Config backends +aws-sdk-dynamodb = "1" +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres"] } + +# Server runtime +tokio = { version = "1", features = ["full"] } +hyper = { version = "1", features = ["full"] } +hyper-util = { version = "0.1", features = ["full"] } +http-body-util = "0.1" +tower = { version = "0.5", features = ["full"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } -[dev-dependencies] -common-s3-headers = "1.0.0" +# Internal crates +s3-proxy-core = { path = "crates/libs/core" } +s3-proxy-auth = { path = "crates/libs/auth" } diff --git a/Dockerfile b/Dockerfile index f8fc9b8..51226ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,49 +1,17 @@ -# Build stage - target x86_64 for ECS Fargate compatibility -# Use bookworm variant to match runtime stage GLIBC version -FROM --platform=linux/amd64 rust:1.90.0-bookworm AS builder +FROM rust:1.82-slim AS builder -# Set environment variables for consistent builds -ENV CARGO_TARGET_DIR=/app/target -ENV RUSTFLAGS="-C target-cpu=x86-64" - -# Copy source code -COPY . /app WORKDIR /app +COPY . . -# Add x86_64 target and build -RUN rustup target add x86_64-unknown-linux-gnu -RUN cargo build --release --target x86_64-unknown-linux-gnu - -# Runtime stage - minimal Debian image -FROM --platform=linux/amd64 debian:bookworm-slim AS runtime - -# Install runtime dependencies (ca-certificates for HTTPS requests) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - && rm -rf /var/lib/apt/lists/* +RUN cargo build --release --package s3-proxy-server --bin s3-proxy -# Create app user for security -RUN groupadd -r appuser && useradd -r -g appuser appuser +FROM debian:bookworm-slim -# Copy the built binary from builder stage -COPY --from=builder /app/target/x86_64-unknown-linux-gnu/release/source-data-proxy /app/source-data-proxy +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* -# Set proper permissions -RUN chown appuser:appuser /app/source-data-proxy && \ - chmod +x /app/source-data-proxy +COPY --from=builder /app/target/release/s3-proxy /usr/local/bin/s3-proxy -# Switch to non-root user -USER appuser - -# Set working directory and expose port -WORKDIR /app EXPOSE 8080 -# Health check endpoint (using root path which returns version info) -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8080/ || exit 1 - -# Run the binary directly -ENTRYPOINT ["/app/source-data-proxy"] +ENTRYPOINT ["s3-proxy"] +CMD ["--config", "/etc/s3-proxy/config.toml", "--listen", "0.0.0.0:8080"] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 16d2966..0000000 --- a/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (c) 2024 Radiant Earth - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f264726..cdb4a7a 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,325 @@ -# Source Cooperative Data Proxy +# s3-proxy-rs -This repository contains the rust application which hosts the Source Cooperative Data Proxy. +A multi-runtime S3 gateway that streams requests to and from backing object stores (S3, MinIO, R2, etc.), providing a unified S3-compliant API with configurable authentication and authorization. -## Getting Started +## Architecture -### Prerequisites +``` +┌──────────────┐ ┌──────────────────────────────────────┐ +│ S3 Clients │────────▶│ s3-proxy-rs │ +│ (aws cli, │ │ │ +│ boto3, │ │ ┌────────────┐ ┌────────────────┐ │ ┌──────────────┐ +│ sdk, etc.) │ │ │ Auth │ │ Config │ │────────▶│ Backend S3 │ +│ │◀────────│ │ (SigV4, │ │ (Static, │ │ │ (AWS, MinIO │ +│ │ │ │ STS, │ │ HTTP API, │ │◀────────│ R2, etc.) │ +│ │ │ │ OIDC) │ │ DynamoDB, │ │ └──────────────┘ +│ │ │ └────────────┘ │ Postgres) │ │ +└──────────────┘ │ └────────────────┘ │ + └──────────────────────────────────────┘ +``` + +### Crate Layout + +``` +crates/ +├── libs/ # Libraries — not directly runnable +│ ├── core/ (s3-proxy-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config providers +│ └── auth/ (s3-proxy-auth) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) +└── runtimes/ # Runnable targets — one per deployment platform + ├── server/ (s3-proxy-server) # Tokio/Hyper for container deployments + └── cf-workers/ (s3-proxy-cf-workers) # Cloudflare Workers for edge deployments +``` + +Libraries define trait abstractions (`BodyStream`, `BackendClient`, `ConfigProvider`, `RequestResolver`). Runtimes implement those traits with platform-native primitives: Hyper/reqwest on the server, JS Fetch API with `ReadableStream` passthrough on Workers. + +The `RequestResolver` trait decouples "what to do with a request" from the proxy handler. A `DefaultResolver` handles standard S3 proxy behavior (parse, auth, authorize via `ConfigProvider`), while custom resolvers like `SourceCoopResolver` can implement entirely different namespace mapping and authorization schemes. + +## Supported Operations + +- `GET` (GetObject) — download files +- `HEAD` (HeadObject) — file metadata +- `PUT` (PutObject) — upload files +- `POST` (CreateMultipartUpload, CompleteMultipartUpload) — multipart uploads +- `PUT` with `partNumber` + `uploadId` — upload individual parts +- `DELETE` with `uploadId` — abort multipart uploads +- `GET` on bucket root — ListBucket (v2) +- STS `AssumeRoleWithWebIdentity` — OIDC token exchange + +## Quick Start + +### Local Development (Docker Compose) + +The fastest way to get a running environment with MinIO as the backing store: + +```bash +docker compose up +``` + +This starts MinIO (`:9000` API, `:9001` console) and a seed job that creates example buckets with test data. Then run the proxy using either runtime: + +```bash +# Option A: native server runtime +cargo run -p s3-proxy-server -- --config config.local.toml --listen 0.0.0.0:8080 + +# Option B: Cloudflare Workers runtime (via Wrangler) +cd crates/runtimes/cf-workers && npx wrangler dev +``` + +Test it: + +```bash +# Anonymous read (port 8080 for server, 8787 for worker) +curl http://localhost:8080/public-data/hello.txt + +# Signed upload with the local dev credential +AWS_ACCESS_KEY_ID=AKLOCAL0000000000001 \ +AWS_SECRET_ACCESS_KEY="localdev/secret/key/00000000000000000000" \ +aws s3 cp ./myfile.txt s3://private-uploads/myfile.txt \ + --endpoint-url http://localhost:8080 + +# Browse MinIO directly +open http://localhost:9001 # user: minioadmin / pass: minioadmin +``` + +The server runtime reads `config.local.toml` (TOML, backend endpoints use `http://localhost:9000`). The worker runtime reads `PROXY_CONFIG` from `crates/runtimes/cf-workers/wrangler.toml` (JSON, same endpoints). + +### Container Deployment + +```bash +# Build +cargo build --release -p s3-proxy-server + +# Run with a config file +./target/release/s3-proxy --config config.toml --listen 0.0.0.0:8080 + +# Or with Docker +docker build -t s3-proxy . +docker run -v ./config.toml:/etc/s3-proxy/config.toml -p 8080:8080 s3-proxy +``` + +### Client Usage + +```bash +# Anonymous access to a public bucket +curl http://localhost:8080/public-data/path/to/file.txt + +# Signed request with aws-cli (using long-lived credentials) +aws s3 cp s3://ml-artifacts/models/latest.pt ./latest.pt \ + --endpoint-url http://localhost:8080 + +# GitHub Actions OIDC → STS → S3 +# Step 1: Exchange OIDC token for temporary credentials +CREDS=$(aws sts assume-role-with-web-identity \ + --role-arn github-actions-deployer \ + --web-identity-token "$ACTIONS_ID_TOKEN" \ + --endpoint-url http://localhost:8080) + +# Step 2: Use temporary credentials to access S3 +AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r .AccessKeyId) \ +AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r .SecretAccessKey) \ +AWS_SESSION_TOKEN=$(echo $CREDS | jq -r .SessionToken) \ +aws s3 cp ./bundle.tar.gz s3://deploy-bundles/releases/v1.2.3.tar.gz \ + --endpoint-url http://localhost:8080 +``` + +## Configuration + +See `config.example.toml` for a full example. The config defines three things: virtual buckets (mapping client-visible names to backing stores), IAM roles (trust policies for OIDC token exchange), and long-lived credentials (static access keys). + +### Configuration Providers + +The proxy supports multiple backends for retrieving configuration, selectable at build time via feature flags or by choosing a different provider at runtime. + +#### Static File (always available) + +```rust +use s3_proxy_core::config::static_file::StaticProvider; + +let provider = StaticProvider::from_file("config.toml")?; +// or +let provider = StaticProvider::from_toml(include_str!("../config.toml"))?; +// or from JSON (useful for Workers env vars) +let provider = StaticProvider::from_json(&json_string)?; +``` + +#### HTTP API (`config-http` feature) + +Fetches config from a centralized REST API. Useful with a control plane service. + +```rust +use s3_proxy_core::config::http::HttpProvider; + +let provider = HttpProvider::new( + "https://config-api.internal:8080".to_string(), + Some("Bearer my-api-token".to_string()), +); +``` + +Expected endpoints: `GET /buckets`, `GET /buckets/{name}`, `GET /roles/{id}`, etc. -- Cargo installed on your local machine -- The AWS CLI installed on your local machine +#### DynamoDB (`config-dynamodb` feature) -### Run Locally +Single-table design with PK/SK pattern. Build with: -To run the data proxy locally, run the following command: +```bash +cargo build -p s3-proxy-server --features s3-proxy-core/config-dynamodb +``` + +```rust +use s3_proxy_core::config::dynamodb::DynamoDbProvider; + +let client = aws_sdk_dynamodb::Client::new(&aws_config); +let provider = DynamoDbProvider::new(client, "s3-proxy-config".to_string()); +``` + +#### PostgreSQL (`config-postgres` feature) + +```bash +cargo build -p s3-proxy-server --features s3-proxy-core/config-postgres +``` + +```rust +use s3_proxy_core::config::postgres::PostgresProvider; + +let pool = sqlx::PgPool::connect("postgres://localhost/s3proxy").await?; +let provider = PostgresProvider::new(pool); +``` + +#### Implementing a Custom Provider + +Implement the `ConfigProvider` trait to plug in your own config backend. Then wrap it in `DefaultResolver` to get standard S3 proxy behavior (path/virtual-host parsing, auth, authorization): + +```rust +use s3_proxy_core::config::ConfigProvider; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::*; + +#[derive(Clone)] +struct MyProvider { /* ... */ } + +impl ConfigProvider for MyProvider { + async fn list_buckets(&self) -> Result, ProxyError> { todo!() } + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { todo!() } + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { todo!() } + async fn get_credential(&self, access_key_id: &str) -> Result, ProxyError> { todo!() } + async fn store_temporary_credential(&self, cred: &TemporaryCredentials) -> Result<(), ProxyError> { todo!() } + async fn get_temporary_credential(&self, access_key_id: &str) -> Result, ProxyError> { todo!() } +} +``` + +#### Implementing a Custom Request Resolver + +For full control over request routing, authentication, and namespace mapping, implement the `RequestResolver` trait directly. This is useful when your URL namespace doesn't map to a simple bucket/key structure, or when authorization is handled by an external service. + +```rust +use s3_proxy_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::BucketConfig; +use http::{Method, HeaderMap}; +use bytes::Bytes; + +#[derive(Clone)] +struct MyResolver { /* ... */ } + +impl RequestResolver for MyResolver { + async fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> Result { + // Parse the path, authenticate, authorize, and return either: + // - ResolvedAction::Proxy { operation, bucket_config, list_rewrite } + // to forward to a backend + // - ResolvedAction::Response { status, headers, body } + // to return a synthetic response (e.g., virtual directory listing) + todo!() + } +} +``` + +The `ResolvedAction::Proxy` variant supports an optional `ListRewrite` for rewriting `` and `` values in S3 list response XML, which is useful when the backend prefix differs from what clients expect. + +Wire it into the proxy handler in your runtime: + +```rust +let resolver = MyResolver::new(/* ... */); +let handler = ProxyHandler::new(backend_client, resolver); +let result = handler.handle_request(method, path, query, &headers, body).await; +``` + +### Caching Configuration + +Wrap any provider with `CachedProvider` to add in-memory TTL-based caching. This is recommended for all network-backed providers. + +```rust +use s3_proxy_core::config::cached::CachedProvider; +use std::time::Duration; +// Wrap any provider with a 5-minute cache +let base = HttpProvider::new("https://config-api.internal".into(), None); +let provider = CachedProvider::new(base, Duration::from_secs(300)); + +// First call hits the backend; subsequent calls within TTL return cached data. +// Temporary credential operations (store/get) bypass the cache. + +// Manual invalidation is also available: +provider.invalidate_all(); +provider.invalidate_bucket("my-bucket"); ``` -./scripts/run.sh + +The cache is thread-safe (`RwLock`-based) and evicts entries lazily on access. Temporary credential writes and reads always go directly to the underlying provider — they're already short-lived and caching them would create security issues with stale session tokens. + +## Authentication Flows + +### 1. Anonymous Access + +Buckets with `anonymous_access = true` serve GET/HEAD/LIST requests without any authentication. Write operations still require credentials. + +### 2. Long-Lived Access Keys + +Static `AccessKeyId`/`SecretAccessKey` pairs stored in the config backend. Clients sign requests using standard AWS SigV4. Each credential has an associated set of access scopes (buckets, prefixes, allowed actions). + +### 3. OIDC → STS → Temporary Credentials + +Modeled after AWS `AssumeRoleWithWebIdentity`. This is the recommended approach for CI/CD workloads. + +**GitHub Actions example workflow:** + +```yaml +jobs: + deploy: + permissions: + id-token: write # Required for OIDC token + steps: + - name: Get OIDC token + id: oidc + run: | + TOKEN=$(curl -s -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.s3proxy.example.com" | jq -r '.value') + echo "token=$TOKEN" >> $GITHUB_OUTPUT + + - name: Assume role via STS + run: | + aws sts assume-role-with-web-identity \ + --role-arn github-actions-deployer \ + --web-identity-token ${{ steps.oidc.outputs.token }} \ + --endpoint-url https://s3proxy.example.com ``` + +The proxy validates the JWT against the OIDC provider's JWKS, checks the trust policy (issuer, audience, subject conditions with glob matching), and mints temporary credentials scoped to the role's allowed buckets/prefixes. + +## Multi-Runtime Design + +The crate workspace separates concerns so the core logic compiles to both native and WASM targets: + +**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. + +**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It implements `BodyStream` with `http_body_util` types and `BackendClient` with reqwest's streaming HTTP client. + +**s3-proxy-cf-workers** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It implements `BodyStream` wrapping JS `ReadableStream` and `BackendClient` using the Fetch API. The critical optimization: for GET requests, the JS `ReadableStream` from the backend response is passed directly to the outbound worker `Response` — bytes never touch Rust/WASM memory. + +## License + +MIT OR Apache-2.0 diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..975222c --- /dev/null +++ b/config.example.toml @@ -0,0 +1,105 @@ +# s3-proxy configuration example +# +# This file defines the virtual buckets, IAM roles, and long-lived +# credentials that the proxy serves. + +# ============================================================================= +# Virtual Buckets +# ============================================================================= + +# A publicly accessible bucket (anonymous reads allowed) +[[buckets]] +name = "public-data" +backend_endpoint = "https://s3.us-east-1.amazonaws.com" +backend_bucket = "my-company-public-assets" +backend_region = "us-east-1" +backend_access_key_id = "AKIAIOSFODNN7EXAMPLE" +backend_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +anonymous_access = true +allowed_roles = [] + +# A private bucket backed by MinIO +[[buckets]] +name = "ml-artifacts" +backend_endpoint = "https://minio.internal:9000" +backend_bucket = "ml-pipeline-artifacts" +backend_prefix = "v2" +backend_region = "us-east-1" +backend_access_key_id = "minioadmin" +backend_secret_access_key = "minioadmin" +anonymous_access = false +allowed_roles = ["github-actions-deployer"] + +# Another private bucket on a different S3-compatible service +[[buckets]] +name = "deploy-bundles" +backend_endpoint = "https://s3.us-west-2.amazonaws.com" +backend_bucket = "prod-deploy-bundles" +backend_region = "us-west-2" +backend_access_key_id = "AKIAI44QH8DHBEXAMPLE" +backend_secret_access_key = "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" +anonymous_access = false +allowed_roles = ["github-actions-deployer", "ci-readonly"] + +# ============================================================================= +# IAM Roles (for STS AssumeRoleWithWebIdentity) +# ============================================================================= + +# Role for GitHub Actions CI/CD pipelines +[[roles]] +role_id = "github-actions-deployer" +name = "GitHub Actions Deploy Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +required_audience = "sts.s3proxy.example.com" +# Only allow specific repos/branches +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", + "repo:myorg/myapp:ref:refs/heads/release/*", + "repo:myorg/infrastructure:*", +] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/", "datasets/"] +actions = ["get_object", "head_object", "put_object", "create_multipart_upload", "upload_part", "complete_multipart_upload"] + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] # full bucket access +actions = ["get_object", "head_object", "put_object", "create_multipart_upload", "upload_part", "complete_multipart_upload"] + +# Read-only role for CI +[[roles]] +role_id = "ci-readonly" +name = "CI Read-Only Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +subject_conditions = ["repo:myorg/*"] +max_session_duration_secs = 1800 + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] + +# ============================================================================= +# Long-Lived Credentials +# ============================================================================= + +# Service account for an internal tool +[[credentials]] +access_key_id = "AKPROXY00000EXAMPLE" +secret_access_key = "proxy/secret/key/EXAMPLE000000000000" +principal_name = "internal-dashboard" +created_at = "2024-01-15T00:00:00Z" +enabled = true + +[[credentials.allowed_scopes]] +bucket = "public-data" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] + +[[credentials.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/production/"] +actions = ["get_object", "head_object"] diff --git a/crates/libs/auth/Cargo.toml b/crates/libs/auth/Cargo.toml new file mode 100644 index 0000000..73ac2a2 --- /dev/null +++ b/crates/libs/auth/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "s3-proxy-auth" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "OIDC/STS authentication for the S3 proxy gateway" + +[dependencies] +s3-proxy-core.workspace = true +async-trait.workspace = true +thiserror.workspace = true +serde.workspace = true +serde_json.workspace = true +chrono.workspace = true +uuid.workspace = true +base64.workspace = true +rsa.workspace = true +sha2.workspace = true +reqwest.workspace = true +tracing.workspace = true diff --git a/crates/libs/auth/README.md b/crates/libs/auth/README.md new file mode 100644 index 0000000..416e7cf --- /dev/null +++ b/crates/libs/auth/README.md @@ -0,0 +1,74 @@ +# s3-proxy-auth + +OIDC token exchange and STS credential minting for the S3 proxy gateway. Implements the `AssumeRoleWithWebIdentity` flow, allowing workloads like GitHub Actions to exchange OIDC JWTs for temporary, scoped S3 credentials. + +## What This Crate Does + +``` +GitHub Actions (or any OIDC provider) + │ + │ JWT (signed by provider) + ▼ +┌─────────────────────────────┐ +│ s3-proxy-auth │ +│ │ +│ 1. Decode JWT header │ +│ 2. Fetch JWKS from issuer │ +│ 3. Verify JWT signature │ +│ 4. Check trust policy: │ +│ - issuer ∈ trusted? │ +│ - audience matches? │ +│ - subject matches glob? │ +│ 5. Mint temporary creds │ +│ (AccessKeyId, │ +│ SecretAccessKey, │ +│ SessionToken) │ +│ 6. Store via ConfigProvider│ +└─────────────────────────────┘ + │ + │ TemporaryCredentials + ▼ +Client signs S3 requests with temp creds +``` + +## Runtime Coupling + +This crate uses `reqwest` for JWKS fetching, which works on both native and WASM targets (`reqwest` compiles to `wasm32-unknown-unknown` using `web-sys` fetch). It does not depend on Tokio directly; the async functions are runtime-agnostic. + +If you need to use a different HTTP client for JWKS fetching (e.g., the Workers Fetch API directly), you'd replace the `fetch_jwks` function in `jwks.rs` or introduce a trait for HTTP fetching. This is a reasonable follow-up if WASM binary size becomes a concern. + +## Module Overview + +``` +src/ +├── lib.rs Entry point: assume_role_with_web_identity(), subject glob matching +├── jwks.rs JWKS fetching, JWK parsing, JWT signature verification +└── sts.rs Temporary credential minting (AccessKeyId/SecretAccessKey/SessionToken) +``` + +## Usage + +Called by the proxy handler when it receives an STS `AssumeRoleWithWebIdentity` request: + +```rust +use s3_proxy_auth::assume_role_with_web_identity; + +let creds = assume_role_with_web_identity( + &config_provider, + "github-actions-deployer", // role ARN + &jwt_token, // OIDC token from the client + Some(3600), // session duration (seconds) +).await?; + +// creds.access_key_id, creds.secret_access_key, creds.session_token +// are returned to the client in an STS XML response. +``` + +## Trust Policies + +Roles define trust policies in the config: + +- **`trusted_oidc_issuers`** — which OIDC providers are accepted (e.g., `https://token.actions.githubusercontent.com`) +- **`required_audience`** — the `aud` claim the JWT must contain +- **`subject_conditions`** — glob patterns matched against the `sub` claim (e.g., `repo:myorg/myrepo:ref:refs/heads/main`, `repo:myorg/*`) +- **`allowed_scopes`** — buckets, prefixes, and actions the minted credentials grant access to diff --git a/crates/libs/auth/src/jwks.rs b/crates/libs/auth/src/jwks.rs new file mode 100644 index 0000000..ede1a68 --- /dev/null +++ b/crates/libs/auth/src/jwks.rs @@ -0,0 +1,179 @@ +//! JWKS fetching and JWT verification. + +use base64::Engine; +use rsa::pkcs1v15::VerifyingKey; +use rsa::signature::Verifier; +use rsa::{BigUint, RsaPublicKey}; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::RoleConfig; +use serde::Deserialize; +use sha2::Sha256; + +#[derive(Debug, Deserialize)] +pub struct JwksResponse { + pub keys: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct JwkKey { + pub kid: String, + pub kty: String, + pub alg: Option, + pub n: Option, + pub e: Option, + #[serde(rename = "use")] + pub use_: Option, +} + +/// Fetch JWKS from an OIDC provider's well-known endpoint. +pub async fn fetch_jwks(issuer: &str) -> Result { + let issuer = issuer.trim_end_matches('/'); + + // First, try the .well-known/openid-configuration endpoint + let config_url = format!("{}/.well-known/openid-configuration", issuer); + let client = reqwest::Client::new(); + + let config_resp = client + .get(&config_url) + .send() + .await + .map_err(|e| ProxyError::InvalidOidcToken(format!("failed to fetch OIDC config: {}", e)))?; + + let config: serde_json::Value = config_resp + .json() + .await + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid OIDC config: {}", e)))?; + + let jwks_uri = config + .get("jwks_uri") + .and_then(|v| v.as_str()) + .ok_or_else(|| ProxyError::InvalidOidcToken("OIDC config missing jwks_uri".into()))?; + + // Fetch the JWKS + let jwks_resp = client + .get(jwks_uri) + .send() + .await + .map_err(|e| ProxyError::InvalidOidcToken(format!("failed to fetch JWKS: {}", e)))?; + + jwks_resp + .json() + .await + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid JWKS: {}", e))) +} + +/// Find a key in the JWKS by key ID. +pub fn find_key<'a>(jwks: &'a JwksResponse, kid: &str) -> Result<&'a JwkKey, ProxyError> { + jwks.keys + .iter() + .find(|k| k.kid == kid) + .ok_or_else(|| ProxyError::InvalidOidcToken(format!("key '{}' not found in JWKS", kid))) +} + +/// Decode a base64url-encoded string (no padding). +fn base64url_decode(input: &str) -> Result, ProxyError> { + base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(input) + .map_err(|e| ProxyError::InvalidOidcToken(format!("base64url decode error: {}", e))) +} + +/// Build an RSA public key from JWK `n` and `e` components. +fn rsa_public_key_from_components(n: &str, e: &str) -> Result { + let n_bytes = base64url_decode(n)?; + let e_bytes = base64url_decode(e)?; + let n_int = BigUint::from_bytes_be(&n_bytes); + let e_int = BigUint::from_bytes_be(&e_bytes); + RsaPublicKey::new(n_int, e_int) + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid RSA key: {}", e))) +} + +/// Verify a JWT using the provided JWK. +pub fn verify_token( + token: &str, + key: &JwkKey, + issuer: &str, + role: &RoleConfig, +) -> Result { + let n = key + .n + .as_ref() + .ok_or_else(|| ProxyError::InvalidOidcToken("JWK missing 'n' component".into()))?; + let e = key + .e + .as_ref() + .ok_or_else(|| ProxyError::InvalidOidcToken("JWK missing 'e' component".into()))?; + + // Split the JWT into parts + let parts: Vec<&str> = token.splitn(3, '.').collect(); + if parts.len() != 3 { + return Err(ProxyError::InvalidOidcToken("malformed JWT".into())); + } + let [header_b64, payload_b64, signature_b64] = [parts[0], parts[1], parts[2]]; + + // Verify the header specifies RS256 + let header_bytes = base64url_decode(header_b64)?; + let header: serde_json::Value = serde_json::from_slice(&header_bytes) + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid JWT header JSON: {}", e)))?; + let alg = header + .get("alg") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if alg != "RS256" { + return Err(ProxyError::InvalidOidcToken(format!( + "unsupported JWT algorithm: {}", + alg + ))); + } + + // Verify the RSA signature + let public_key = rsa_public_key_from_components(n, e)?; + let verifying_key = VerifyingKey::::new(public_key); + let signature_bytes = base64url_decode(signature_b64)?; + let signature = rsa::pkcs1v15::Signature::try_from(signature_bytes.as_slice()) + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid signature: {}", e)))?; + let signed_content = format!("{}.{}", header_b64, payload_b64); + verifying_key + .verify(signed_content.as_bytes(), &signature) + .map_err(|e| ProxyError::InvalidOidcToken(format!("JWT signature verification failed: {}", e)))?; + + // Decode and validate claims + let payload_bytes = base64url_decode(payload_b64)?; + let claims: serde_json::Value = serde_json::from_slice(&payload_bytes) + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid JWT payload JSON: {}", e)))?; + + // Validate issuer + let token_issuer = claims.get("iss").and_then(|v| v.as_str()).unwrap_or(""); + if token_issuer != issuer { + return Err(ProxyError::InvalidOidcToken(format!( + "issuer mismatch: expected {}, got {}", + issuer, token_issuer + ))); + } + + // Validate audience if required + if let Some(ref required_aud) = role.required_audience { + let aud_valid = match claims.get("aud") { + Some(serde_json::Value::String(aud)) => aud == required_aud, + Some(serde_json::Value::Array(auds)) => { + auds.iter().any(|a| a.as_str() == Some(required_aud.as_str())) + } + _ => false, + }; + if !aud_valid { + return Err(ProxyError::InvalidOidcToken(format!( + "audience mismatch: expected {}", + required_aud + ))); + } + } + + // Validate expiration + if let Some(exp) = claims.get("exp").and_then(|v| v.as_i64()) { + let now = chrono::Utc::now().timestamp(); + if now > exp { + return Err(ProxyError::InvalidOidcToken("token has expired".into())); + } + } + + Ok(claims) +} diff --git a/crates/libs/auth/src/lib.rs b/crates/libs/auth/src/lib.rs new file mode 100644 index 0000000..f16c664 --- /dev/null +++ b/crates/libs/auth/src/lib.rs @@ -0,0 +1,174 @@ +//! OIDC/STS authentication for the S3 proxy gateway. +//! +//! This crate implements the `AssumeRoleWithWebIdentity` STS API, allowing +//! workloads like GitHub Actions to exchange OIDC tokens for temporary S3 +//! credentials scoped to specific buckets and prefixes. +//! +//! # Flow +//! +//! 1. Client obtains a JWT from their OIDC provider (e.g., GitHub Actions ID token) +//! 2. Client calls `AssumeRoleWithWebIdentity` with the JWT and desired role +//! 3. This crate validates the JWT against the OIDC provider's JWKS +//! 4. Checks trust policy (issuer, audience, subject conditions) +//! 5. Mints temporary credentials (AccessKeyId/SecretAccessKey/SessionToken) +//! 6. Returns credentials to the client +//! +//! The client then uses these credentials to sign S3 requests normally. + +pub mod jwks; +pub mod sts; + +use base64::Engine; +use s3_proxy_core::config::ConfigProvider; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::TemporaryCredentials; + +/// Decode JWT header and claims without signature verification. +fn jwt_decode_unverified( + token: &str, +) -> Result<(serde_json::Value, serde_json::Value), ProxyError> { + let mut parts = token.splitn(3, '.'); + let header_b64 = parts + .next() + .ok_or_else(|| ProxyError::InvalidOidcToken("malformed JWT".into()))?; + let payload_b64 = parts + .next() + .ok_or_else(|| ProxyError::InvalidOidcToken("malformed JWT".into()))?; + + let decode = |s: &str| -> Result { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(s) + .map_err(|e| ProxyError::InvalidOidcToken(format!("base64url decode error: {}", e)))?; + serde_json::from_slice(&bytes) + .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid JWT JSON: {}", e))) + }; + + Ok((decode(header_b64)?, decode(payload_b64)?)) +} + +/// Validate an OIDC token and mint temporary credentials. +pub async fn assume_role_with_web_identity( + config: &C, + role_arn: &str, + web_identity_token: &str, + duration_seconds: Option, +) -> Result { + // Look up the role + let role = config + .get_role(role_arn) + .await? + .ok_or_else(|| ProxyError::RoleNotFound(role_arn.to_string()))?; + + // Decode the JWT header and claims without verification to extract issuer and kid + let (header, insecure_claims) = jwt_decode_unverified(web_identity_token)?; + + let issuer = insecure_claims + .get("iss") + .and_then(|v| v.as_str()) + .ok_or_else(|| ProxyError::InvalidOidcToken("missing iss claim".into()))?; + + // Verify the issuer is trusted + if !role.trusted_oidc_issuers.iter().any(|i| i == issuer) { + return Err(ProxyError::InvalidOidcToken(format!( + "untrusted issuer: {}", + issuer + ))); + } + + // Fetch JWKS and verify the token + let jwks = jwks::fetch_jwks(issuer).await?; + let kid = header + .get("kid") + .and_then(|v| v.as_str()) + .ok_or_else(|| ProxyError::InvalidOidcToken("JWT missing kid".into()))?; + + let key = jwks::find_key(&jwks, kid)?; + let claims = jwks::verify_token(web_identity_token, key, issuer, &role)?; + + // Check subject conditions + let subject = claims + .get("sub") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + if !role.subject_conditions.is_empty() { + let matches = role + .subject_conditions + .iter() + .any(|pattern| subject_matches(subject, pattern)); + if !matches { + return Err(ProxyError::InvalidOidcToken(format!( + "subject '{}' does not match any conditions", + subject + ))); + } + } + + // Mint temporary credentials + let duration = duration_seconds + .unwrap_or(3600) + .min(role.max_session_duration_secs); + + let creds = sts::mint_temporary_credentials(&role, subject, duration); + + // Store them + config.store_temporary_credential(&creds).await?; + + Ok(creds) +} + +/// Simple glob-style matching for subject conditions. +/// Supports `*` as a wildcard for any sequence of characters. +fn subject_matches(subject: &str, pattern: &str) -> bool { + if pattern == "*" { + return true; + } + + let parts: Vec<&str> = pattern.split('*').collect(); + if parts.len() == 1 { + return subject == pattern; + } + + let mut remaining = subject; + + // First part must be a prefix + if !parts[0].is_empty() { + if !remaining.starts_with(parts[0]) { + return false; + } + remaining = &remaining[parts[0].len()..]; + } + + // Middle parts must appear in order + for part in &parts[1..parts.len() - 1] { + if part.is_empty() { + continue; + } + match remaining.find(part) { + Some(idx) => remaining = &remaining[idx + part.len()..], + None => return false, + } + } + + // Last part must be a suffix + let last = parts.last().unwrap(); + if !last.is_empty() { + return remaining.ends_with(last); + } + + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_subject_matching() { + assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/repo:*")); + assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "*")); + assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/repo:ref:refs/heads/main")); + assert!(!subject_matches("repo:org/repo:ref:refs/heads/main", "repo:other/*")); + assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/*:ref:refs/heads/*")); + } +} diff --git a/crates/libs/auth/src/sts.rs b/crates/libs/auth/src/sts.rs new file mode 100644 index 0000000..249f465 --- /dev/null +++ b/crates/libs/auth/src/sts.rs @@ -0,0 +1,52 @@ +//! STS credential minting. + +use chrono::{Duration, Utc}; +use s3_proxy_core::types::{RoleConfig, TemporaryCredentials}; +use uuid::Uuid; + +/// Mint a new set of temporary credentials for an assumed role. +pub fn mint_temporary_credentials( + role: &RoleConfig, + source_identity: &str, + duration_seconds: u64, +) -> TemporaryCredentials { + let access_key_id = format!("ASIA{}", generate_random_id(16)); + let secret_access_key = generate_random_id(40); + let session_token = generate_session_token(); + + let expiration = Utc::now() + Duration::seconds(duration_seconds as i64); + + TemporaryCredentials { + access_key_id, + secret_access_key, + session_token, + expiration, + allowed_scopes: role.allowed_scopes.clone(), + assumed_role_id: role.role_id.clone(), + source_identity: source_identity.to_string(), + } +} + +fn generate_random_id(len: usize) -> String { + use base64::Engine; + let bytes: Vec = (0..len).map(|_| rand_byte()).collect(); + let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&bytes); + // Take only alphanumeric chars to match AWS key format + encoded + .chars() + .filter(|c| c.is_alphanumeric()) + .take(len) + .collect() +} + +fn generate_session_token() -> String { + // Real AWS session tokens are much longer; this is a simplified version + let id = Uuid::new_v4(); + format!("FwoGZXIvYXdzE{}", id.to_string().replace('-', "")) +} + +/// Simple random byte using UUID as entropy source (avoids extra deps). +fn rand_byte() -> u8 { + let id = Uuid::new_v4(); + id.as_bytes()[0] +} diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml new file mode 100644 index 0000000..8144a5c --- /dev/null +++ b/crates/libs/core/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "s3-proxy-core" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Runtime-agnostic core library for the S3 proxy gateway" + +[features] +default = [] +config-dynamodb = ["aws-sdk-dynamodb", "tokio"] +config-postgres = ["sqlx"] +config-http = ["reqwest"] + +[dependencies] +async-trait.workspace = true +thiserror.workspace = true +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +bytes.workspace = true +http.workspace = true +chrono.workspace = true +uuid.workspace = true +base64.workspace = true +hex.workspace = true +url.workspace = true +hmac.workspace = true +sha2.workspace = true +quick-xml.workspace = true +tracing.workspace = true + +# Optional config backend deps +aws-sdk-dynamodb = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } +sqlx = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md new file mode 100644 index 0000000..b0c7e1f --- /dev/null +++ b/crates/libs/core/README.md @@ -0,0 +1,117 @@ +# s3-proxy-core + +Runtime-agnostic core library for the S3 proxy gateway. This crate contains all business logic — S3 request parsing, SigV4 signing/verification, authorization, configuration retrieval, and the proxy handler — without depending on any async runtime. + +## Why This Crate Exists Separately + +The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in containers and Cloudflare Workers on the edge. These runtimes have incompatible stream types, HTTP primitives, and threading models (multi-threaded vs single-threaded WASM). By keeping the core free of runtime dependencies, it compiles cleanly to both `x86_64-unknown-linux-gnu` and `wasm32-unknown-unknown`. + +## Key Abstractions + +The core defines four trait boundaries that runtime crates implement: + +**`BodyStream`** — A streaming body type. The core almost never reads body bytes; it passes them through opaquely from client to backend and back. Each runtime provides its own type (Hyper's `Body`, JS `ReadableStream`, etc.). + +**`BackendClient`** — Makes signed outbound HTTP requests to backing object stores. The server runtime uses `reqwest`; the worker runtime uses the JS Fetch API. + +**`ConfigProvider`** — Retrieves bucket, role, and credential configuration. Ships with four implementations behind feature flags: + +| Provider | Feature | Use Case | +|----------|---------|----------| +| `StaticProvider` | *(always)* | TOML/JSON files, baked-in config | +| `HttpProvider` | `config-http` | Centralized config REST API | +| `DynamoDbProvider` | `config-dynamodb` | AWS-native deployments | +| `PostgresProvider` | `config-postgres` | Database-backed config | + +Any provider can be wrapped with `CachedProvider` for in-memory TTL caching. + +**`RequestResolver`** — Decides what to do with an incoming request. Given an HTTP method, path, query, and headers, a resolver returns a `ResolvedAction`: either forward to a backend (`Proxy`) or return a synthetic response (`Response`). This decouples URL namespace mapping, authentication, and authorization from the proxy handler itself. + +`DefaultResolver` implements the standard S3 proxy flow: parse the S3 operation, look up the bucket in config, authenticate via SigV4, and authorize. Custom resolvers (like the Source Cooperative resolver in `cf-workers`) can implement entirely different routing and auth schemes. + +## Module Overview + +``` +src/ +├── auth.rs SigV4 verification, identity resolution, authorization +├── backend.rs BackendClient trait, S3RequestSigner, outbound SigV4 signing +├── config/ +│ ├── mod.rs ConfigProvider trait definition +│ ├── cached.rs TTL caching wrapper for any provider +│ ├── static_file.rs TOML/JSON file provider +│ ├── http.rs REST API provider (feature: config-http) +│ ├── dynamodb.rs DynamoDB provider (feature: config-dynamodb) +│ └── postgres.rs PostgreSQL provider (feature: config-postgres) +├── error.rs ProxyError with S3-compatible error codes +├── proxy.rs ProxyHandler — the main request handler +├── resolver.rs RequestResolver trait, ResolvedAction, DefaultResolver +├── s3/ +│ ├── request.rs Parse incoming HTTP → S3Operation enum +│ ├── response.rs Serialize S3 XML responses +│ └── list_rewrite.rs Rewrite / values in list response XML +├── stream.rs BodyStream trait +└── types.rs BucketConfig, RoleConfig, StoredCredential, etc. +``` + +## Usage + +This crate is not used directly. Runtime crates (`s3-proxy-server`, `s3-proxy-cf-workers`) depend on it and provide concrete trait implementations. If you're building a custom runtime integration, depend on this crate and implement `BodyStream`, `BackendClient`, and optionally `ConfigProvider` or `RequestResolver`. + +### Standard usage with a ConfigProvider + +Wrap your config provider in `DefaultResolver` for standard S3 proxy behavior (path/virtual-host parsing, SigV4 auth, scope-based authorization): + +```rust +use s3_proxy_core::proxy::ProxyHandler; +use s3_proxy_core::resolver::DefaultResolver; +use s3_proxy_core::config::static_file::StaticProvider; + +let backend_client = MyBackendClient::new(); +let config = StaticProvider::from_file("config.toml")?; +let resolver = DefaultResolver::new(config, Some("s3.example.com".into())); + +let handler = ProxyHandler::new(backend_client, resolver); + +// In your HTTP handler: +let result = handler.handle_request(method, path, query, &headers, body).await; +// Convert `result` (ProxyResult) to your runtime's HTTP response. +``` + +### Custom resolver + +For non-standard URL namespaces or external auth, implement `RequestResolver` directly: + +```rust +use s3_proxy_core::resolver::{RequestResolver, ResolvedAction}; +use s3_proxy_core::error::ProxyError; + +#[derive(Clone)] +struct MyResolver { /* ... */ } + +impl RequestResolver for MyResolver { + async fn resolve( + &self, + method: &http::Method, + path: &str, + query: Option<&str>, + headers: &http::HeaderMap, + ) -> Result { + // Your custom routing, auth, and authorization logic here. + // Return ResolvedAction::Proxy { .. } to forward to a backend, + // or ResolvedAction::Response { .. } for synthetic responses. + todo!() + } +} + +let handler = ProxyHandler::new(backend_client, MyResolver::new()); +``` + +See `s3-proxy-cf-workers/src/source_resolver.rs` for a real-world example that maps a `/{account}/{repo}/{key}` namespace to dynamically-resolved S3 backends with external API authorization. + +## Feature Flags + +All optional — the default build has zero network dependencies: + +- `config-http` — enables `HttpProvider` (adds `reqwest`) +- `config-dynamodb` — enables `DynamoDbProvider` (adds `aws-sdk-dynamodb`, `tokio`) +- `config-postgres` — enables `PostgresProvider` (adds `sqlx`) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs new file mode 100644 index 0000000..5d26a2f --- /dev/null +++ b/crates/libs/core/src/auth.rs @@ -0,0 +1,295 @@ +//! Authentication and authorization. +//! +//! Handles: +//! - SigV4 request verification (incoming requests from clients) +//! - Identity resolution (mapping access key → principal) +//! - Authorization (checking if an identity can perform an operation) + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{Action, ResolvedIdentity, S3Operation}; +use hmac::{Hmac, Mac}; +use http::HeaderMap; +use sha2::{Digest, Sha256}; + +type HmacSha256 = Hmac; + +/// Parsed SigV4 Authorization header. +#[derive(Debug, Clone)] +pub struct SigV4Auth { + pub access_key_id: String, + pub date_stamp: String, + pub region: String, + pub service: String, + pub signed_headers: Vec, + pub signature: String, +} + +/// Parse a SigV4 Authorization header. +/// +/// Format: `AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, +/// SignedHeaders=host;x-amz-date, Signature=abcdef...` +pub fn parse_sigv4_auth(auth_header: &str) -> Result { + let auth_header = auth_header + .strip_prefix("AWS4-HMAC-SHA256 ") + .ok_or_else(|| ProxyError::InvalidRequest("invalid auth scheme".into()))?; + + let mut credential = None; + let mut signed_headers = None; + let mut signature = None; + + for part in auth_header.split(", ") { + if let Some(val) = part.strip_prefix("Credential=") { + credential = Some(val); + } else if let Some(val) = part.strip_prefix("SignedHeaders=") { + signed_headers = Some(val); + } else if let Some(val) = part.strip_prefix("Signature=") { + signature = Some(val); + } + } + + let credential = credential.ok_or_else(|| ProxyError::InvalidRequest("missing Credential".into()))?; + let signed_headers = signed_headers.ok_or_else(|| ProxyError::InvalidRequest("missing SignedHeaders".into()))?; + let signature = signature.ok_or_else(|| ProxyError::InvalidRequest("missing Signature".into()))?; + + // Parse credential: AKID/date/region/service/aws4_request + let cred_parts: Vec<&str> = credential.split('/').collect(); + if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { + return Err(ProxyError::InvalidRequest("malformed credential scope".into())); + } + + Ok(SigV4Auth { + access_key_id: cred_parts[0].to_string(), + date_stamp: cred_parts[1].to_string(), + region: cred_parts[2].to_string(), + service: cred_parts[3].to_string(), + signed_headers: signed_headers.split(';').map(String::from).collect(), + signature: signature.to_string(), + }) +} + +/// Verify a SigV4 signature against a known secret key. +pub fn verify_sigv4_signature( + method: &http::Method, + uri_path: &str, + query_string: &str, + headers: &HeaderMap, + auth: &SigV4Auth, + secret_access_key: &str, + payload_hash: &str, +) -> Result { + // Reconstruct canonical request + let canonical_headers: String = auth + .signed_headers + .iter() + .map(|name| { + let value = headers + .get(name.as_str()) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .trim(); + format!("{}:{}\n", name, value) + }) + .collect(); + + let signed_headers_str = auth.signed_headers.join(";"); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, uri_path, query_string, canonical_headers, signed_headers_str, payload_hash + ); + + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + + let credential_scope = format!( + "{}/{}/{}/aws4_request", + auth.date_stamp, auth.region, auth.service + ); + + let amz_date = headers + .get("x-amz-date") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, canonical_request_hash + ); + + // Derive signing key + let k_date = hmac_sha256( + format!("AWS4{}", secret_access_key).as_bytes(), + auth.date_stamp.as_bytes(), + )?; + let k_region = hmac_sha256(&k_date, auth.region.as_bytes())?; + let k_service = hmac_sha256(&k_region, auth.service.as_bytes())?; + let signing_key = hmac_sha256(&k_service, b"aws4_request")?; + + let expected_signature = hex::encode(hmac_sha256(&signing_key, string_to_sign.as_bytes())?); + + // Constant-time comparison + Ok(constant_time_eq( + expected_signature.as_bytes(), + auth.signature.as_bytes(), + )) +} + +fn hmac_sha256(key: &[u8], data: &[u8]) -> Result, ProxyError> { + let mut mac = + HmacSha256::new_from_slice(key).map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(data); + Ok(mac.finalize().into_bytes().to_vec()) +} + +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + a.iter() + .zip(b.iter()) + .fold(0u8, |acc, (x, y)| acc | (x ^ y)) + == 0 +} + +/// Resolve the identity of an incoming request. +/// +/// Checks the Authorization header and resolves it against the config provider. +pub async fn resolve_identity( + headers: &HeaderMap, + config: &C, +) -> Result { + let auth_header = match headers.get("authorization").and_then(|v| v.to_str().ok()) { + Some(h) => h, + None => return Ok(ResolvedIdentity::Anonymous), + }; + + let sig = parse_sigv4_auth(auth_header)?; + + // Check for temporary credentials first (session token present) + if headers.get("x-amz-security-token").is_some() { + if let Some(temp_cred) = config.get_temporary_credential(&sig.access_key_id).await? { + // Verify session token matches + let session_token = headers + .get("x-amz-security-token") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + if session_token != temp_cred.session_token { + return Err(ProxyError::AccessDenied); + } + return Ok(ResolvedIdentity::Temporary { + credentials: temp_cred, + }); + } + return Err(ProxyError::ExpiredCredentials); + } + + // Check long-lived credentials + if let Some(cred) = config.get_credential(&sig.access_key_id).await? { + if !cred.enabled { + return Err(ProxyError::AccessDenied); + } + if let Some(expires) = cred.expires_at { + if expires <= chrono::Utc::now() { + return Err(ProxyError::ExpiredCredentials); + } + } + return Ok(ResolvedIdentity::LongLived { credential: cred }); + } + + Err(ProxyError::AccessDenied) +} + +/// Check if a resolved identity is authorized to perform an operation. +pub fn authorize( + identity: &ResolvedIdentity, + operation: &S3Operation, + bucket_config: &crate::types::BucketConfig, +) -> Result<(), ProxyError> { + // Anonymous access check + if matches!(identity, ResolvedIdentity::Anonymous) { + if bucket_config.anonymous_access { + // Anonymous users can only read + let action = operation_to_action(operation); + if matches!(action, Action::GetObject | Action::HeadObject | Action::ListBucket) { + return Ok(()); + } + } + return Err(ProxyError::AccessDenied); + } + + let scopes = match identity { + ResolvedIdentity::Anonymous => unreachable!(), + ResolvedIdentity::LongLived { credential } => &credential.allowed_scopes, + ResolvedIdentity::Temporary { credentials } => &credentials.allowed_scopes, + }; + + let action = operation_to_action(operation); + let (bucket, key) = operation_bucket_key(operation); + + // Check if any scope grants access + let authorized = scopes.iter().any(|scope| { + if scope.bucket != bucket { + return false; + } + if !scope.actions.contains(&action) { + return false; + } + // Check prefix restrictions + if scope.prefixes.is_empty() { + return true; // Full bucket access + } + scope + .prefixes + .iter() + .any(|prefix| key.starts_with(prefix)) + }); + + if authorized { + Ok(()) + } else { + Err(ProxyError::AccessDenied) + } +} + +fn operation_to_action(op: &S3Operation) -> Action { + match op { + S3Operation::GetObject { .. } => Action::GetObject, + S3Operation::HeadObject { .. } => Action::HeadObject, + S3Operation::PutObject { .. } => Action::PutObject, + S3Operation::ListBucket { .. } => Action::ListBucket, + S3Operation::CreateMultipartUpload { .. } => Action::CreateMultipartUpload, + S3Operation::UploadPart { .. } => Action::UploadPart, + S3Operation::CompleteMultipartUpload { .. } => Action::CompleteMultipartUpload, + S3Operation::AbortMultipartUpload { .. } => Action::AbortMultipartUpload, + S3Operation::ListBuckets => Action::ListBucket, // Treated as a list operation + S3Operation::AssumeRoleWithWebIdentity { .. } => Action::GetObject, // STS is handled separately + } +} + +fn operation_bucket_key(op: &S3Operation) -> (String, String) { + match op { + S3Operation::GetObject { bucket, key } + | S3Operation::HeadObject { bucket, key } + | S3Operation::PutObject { bucket, key } + | S3Operation::CreateMultipartUpload { bucket, key } + | S3Operation::UploadPart { bucket, key, .. } + | S3Operation::CompleteMultipartUpload { bucket, key, .. } + | S3Operation::AbortMultipartUpload { bucket, key, .. } => { + (bucket.clone(), key.clone()) + } + S3Operation::ListBucket { bucket, raw_query } => { + // Extract prefix from raw query for authorization checks + let prefix = raw_query + .as_deref() + .and_then(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "prefix") + .map(|(_, v)| v.to_string()) + }) + .unwrap_or_default(); + (bucket.clone(), prefix) + } + S3Operation::ListBuckets => (String::new(), String::new()), + S3Operation::AssumeRoleWithWebIdentity { .. } => (String::new(), String::new()), + } +} diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs new file mode 100644 index 0000000..958d535 --- /dev/null +++ b/crates/libs/core/src/backend.rs @@ -0,0 +1,192 @@ +//! Backend client abstraction for making signed requests to backing object stores. + +use crate::error::ProxyError; +use crate::maybe_send::{MaybeSend, MaybeSync}; +use crate::stream::BodyStream; +use http::HeaderMap; +use std::future::Future; + +/// A fully prepared request to send to a backend object store. +#[derive(Debug)] +pub struct BackendRequest { + pub method: http::Method, + pub url: String, + pub headers: HeaderMap, + pub body: B, +} + +/// The response from a backend object store. +pub struct BackendResponse { + pub status: u16, + pub headers: HeaderMap, + pub body: B, +} + +/// Trait for making outbound HTTP requests to backing object stores. +/// +/// Each runtime provides its own implementation: +/// - Server runtime: uses `hyper` client with native async streaming +/// - Worker runtime: uses the Fetch API, keeping JS `ReadableStream` intact +/// +/// The body type `B` is the runtime's native stream type. This ensures +/// zero-copy passthrough: the proxy never materializes the full response +/// body in memory. +pub trait BackendClient: MaybeSend + MaybeSync + 'static { + type Body: BodyStream; + + fn send_request( + &self, + request: BackendRequest, + ) -> impl Future, ProxyError>> + MaybeSend; +} + +/// Helper to build a signed URL + headers for an outbound request to S3. +pub struct S3RequestSigner { + pub access_key_id: String, + pub secret_access_key: String, + pub region: String, + pub service: String, +} + +impl S3RequestSigner { + pub fn new( + access_key_id: String, + secret_access_key: String, + region: String, + ) -> Self { + Self { + access_key_id, + secret_access_key, + region, + service: "s3".to_string(), + } + } + + /// Sign an outbound request using AWS SigV4. + /// + /// This adds Authorization, x-amz-date, x-amz-content-sha256, and Host + /// headers to the provided header map. + pub fn sign_request( + &self, + method: &http::Method, + url: &url::Url, + headers: &mut HeaderMap, + payload_hash: &str, + ) -> Result<(), ProxyError> { + use chrono::Utc; + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let now = Utc::now(); + let date_stamp = now.format("%Y%m%d").to_string(); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + + // Set required headers + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + + let host = url + .host_str() + .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; + let host_header = if let Some(port) = url.port() { + format!("{}:{}", host, port) + } else { + host.to_string() + }; + headers.insert("host", host_header.parse().unwrap()); + + // Canonical request + let canonical_uri = url.path(); + let canonical_querystring = url.query().unwrap_or(""); + + let mut signed_header_names: Vec<&str> = headers + .keys() + .map(|k| k.as_str()) + .collect(); + signed_header_names.sort(); + + let canonical_headers: String = signed_header_names + .iter() + .map(|k| { + let v = headers + .get(*k) + .unwrap() + .to_str() + .unwrap_or("") + .trim(); + format!("{}:{}\n", k, v) + }) + .collect(); + + let signed_headers = signed_header_names.join(";"); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, canonical_uri, canonical_querystring, canonical_headers, signed_headers, payload_hash + ); + + // String to sign + let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, self.region, self.service); + + use sha2::Digest; + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, canonical_request_hash + ); + + // Signing key + type HmacSha256 = Hmac; + + let mut mac = HmacSha256::new_from_slice( + format!("AWS4{}", self.secret_access_key).as_bytes(), + ) + .map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(date_stamp.as_bytes()); + let k_date = mac.finalize().into_bytes(); + + let mut mac = HmacSha256::new_from_slice(&k_date) + .map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(self.region.as_bytes()); + let k_region = mac.finalize().into_bytes(); + + let mut mac = HmacSha256::new_from_slice(&k_region) + .map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(self.service.as_bytes()); + let k_service = mac.finalize().into_bytes(); + + let mut mac = HmacSha256::new_from_slice(&k_service) + .map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(b"aws4_request"); + let signing_key = mac.finalize().into_bytes(); + + // Signature + let mut mac = HmacSha256::new_from_slice(&signing_key) + .map_err(|e| ProxyError::Internal(e.to_string()))?; + mac.update(string_to_sign.as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + + // Authorization header + let auth_header = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + self.access_key_id, credential_scope, signed_headers, signature + ); + headers.insert("authorization", auth_header.parse().unwrap()); + + Ok(()) + } +} + +/// Hash a payload for SigV4. For streaming/unsigned payloads, use the +/// special sentinel value. +pub fn hash_payload(payload: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(payload)) +} + +/// The SigV4 sentinel for unsigned payloads (used with streaming uploads). +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; + +/// The SigV4 sentinel for streaming payloads. +pub const STREAMING_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; diff --git a/crates/libs/core/src/config/cached.rs b/crates/libs/core/src/config/cached.rs new file mode 100644 index 0000000..ede9fb2 --- /dev/null +++ b/crates/libs/core/src/config/cached.rs @@ -0,0 +1,222 @@ +//! Caching wrapper for any [`ConfigProvider`]. +//! +//! Adds in-memory TTL-based caching over a delegate provider. This is +//! recommended for network-backed providers (HTTP, DynamoDB, Postgres) +//! to reduce latency and load on the config backend. +//! +//! # Example +//! +//! ```rust,ignore +//! use s3_proxy_core::config::cached::CachedProvider; +//! use std::time::Duration; +//! +//! // Wrap any provider with a 5-minute cache +//! let provider = CachedProvider::new(my_http_provider, Duration::from_secs(300)); +//! +//! // First call hits the backend +//! let bucket = provider.get_bucket("my-bucket").await?; +//! +//! // Subsequent calls within 5 minutes return the cached value +//! let bucket_again = provider.get_bucket("my-bucket").await?; +//! ``` + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +/// A cache entry with a value and expiration time. +#[derive(Clone)] +struct CacheEntry { + value: T, + inserted_at: Instant, +} + +impl CacheEntry { + fn is_expired(&self, ttl: Duration) -> bool { + self.inserted_at.elapsed() > ttl + } +} + +/// Wraps a [`ConfigProvider`] with in-memory TTL-based caching. +/// +/// Thread-safe via `RwLock`. Cache entries are evicted lazily on access. +/// Temporary credential storage is delegated directly to the underlying +/// provider (no caching for writes). +#[derive(Clone)] +pub struct CachedProvider

{ + inner: P, + cache: Arc, + ttl: Duration, +} + +struct CacheState { + buckets_list: RwLock>>>, + buckets: RwLock>>>, + roles: RwLock>>>, + credentials: RwLock>>>, +} + +impl CachedProvider

{ + /// Create a new caching wrapper with the given TTL. + pub fn new(inner: P, ttl: Duration) -> Self { + Self { + inner, + cache: Arc::new(CacheState { + buckets_list: RwLock::new(None), + buckets: RwLock::new(HashMap::new()), + roles: RwLock::new(HashMap::new()), + credentials: RwLock::new(HashMap::new()), + }), + ttl, + } + } + + /// Invalidate all cached entries. + pub fn invalidate_all(&self) { + if let Ok(mut lock) = self.cache.buckets_list.write() { + *lock = None; + } + if let Ok(mut lock) = self.cache.buckets.write() { + lock.clear(); + } + if let Ok(mut lock) = self.cache.roles.write() { + lock.clear(); + } + if let Ok(mut lock) = self.cache.credentials.write() { + lock.clear(); + } + } + + /// Invalidate a specific bucket entry. + pub fn invalidate_bucket(&self, name: &str) { + if let Ok(mut lock) = self.cache.buckets.write() { + lock.remove(name); + } + // Also invalidate the list since it may contain stale data + if let Ok(mut lock) = self.cache.buckets_list.write() { + *lock = None; + } + } +} + +impl ConfigProvider for CachedProvider

{ + async fn list_buckets(&self) -> Result, ProxyError> { + // Check cache + if let Ok(lock) = self.cache.buckets_list.read() { + if let Some(entry) = &*lock { + if !entry.is_expired(self.ttl) { + return Ok(entry.value.clone()); + } + } + } + + // Cache miss — fetch from inner + let result = self.inner.list_buckets().await?; + + if let Ok(mut lock) = self.cache.buckets_list.write() { + *lock = Some(CacheEntry { + value: result.clone(), + inserted_at: Instant::now(), + }); + } + + Ok(result) + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + // Check cache + if let Ok(lock) = self.cache.buckets.read() { + if let Some(entry) = lock.get(name) { + if !entry.is_expired(self.ttl) { + return Ok(entry.value.clone()); + } + } + } + + let result = self.inner.get_bucket(name).await?; + + if let Ok(mut lock) = self.cache.buckets.write() { + lock.insert( + name.to_string(), + CacheEntry { + value: result.clone(), + inserted_at: Instant::now(), + }, + ); + } + + Ok(result) + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + if let Ok(lock) = self.cache.roles.read() { + if let Some(entry) = lock.get(role_id) { + if !entry.is_expired(self.ttl) { + return Ok(entry.value.clone()); + } + } + } + + let result = self.inner.get_role(role_id).await?; + + if let Ok(mut lock) = self.cache.roles.write() { + lock.insert( + role_id.to_string(), + CacheEntry { + value: result.clone(), + inserted_at: Instant::now(), + }, + ); + } + + Ok(result) + } + + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + if let Ok(lock) = self.cache.credentials.read() { + if let Some(entry) = lock.get(access_key_id) { + if !entry.is_expired(self.ttl) { + return Ok(entry.value.clone()); + } + } + } + + let result = self.inner.get_credential(access_key_id).await?; + + if let Ok(mut lock) = self.cache.credentials.write() { + lock.insert( + access_key_id.to_string(), + CacheEntry { + value: result.clone(), + inserted_at: Instant::now(), + }, + ); + } + + Ok(result) + } + + /// Temporary credential writes bypass the cache and go directly to + /// the underlying provider. + async fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> Result<(), ProxyError> { + self.inner.store_temporary_credential(cred).await + } + + /// Temporary credential reads also bypass the cache — they're already + /// short-lived and we don't want stale session tokens. + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + self.inner.get_temporary_credential(access_key_id).await + } +} diff --git a/crates/libs/core/src/config/dynamodb.rs b/crates/libs/core/src/config/dynamodb.rs new file mode 100644 index 0000000..4230df1 --- /dev/null +++ b/crates/libs/core/src/config/dynamodb.rs @@ -0,0 +1,238 @@ +//! DynamoDB-backed configuration provider. +//! +//! Stores configuration in DynamoDB tables. Designed for AWS-native +//! deployments where DynamoDB is readily available. +//! +//! # Table Schema +//! +//! Uses a single-table design with the following layout: +//! +//! | PK | SK | Attributes | +//! |----|----|------------| +//! | `BUCKET#{name}` | `CONFIG` | BucketConfig fields | +//! | `ROLE#{role_id}` | `CONFIG` | RoleConfig fields | +//! | `CRED#{access_key_id}` | `LONG_LIVED` | StoredCredential fields | +//! | `CRED#{access_key_id}` | `TEMPORARY` | TemporaryCredentials fields (with TTL) | +//! +//! # Example +//! +//! ```rust,ignore +//! use s3_proxy_core::config::dynamodb::DynamoDbProvider; +//! use aws_sdk_dynamodb::Client; +//! +//! let sdk_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; +//! let client = Client::new(&sdk_config); +//! let provider = DynamoDbProvider::new(client, "s3-proxy-config".to_string()); +//! ``` + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use aws_sdk_dynamodb::types::AttributeValue; +use aws_sdk_dynamodb::Client; +use std::sync::Arc; + +/// Configuration provider backed by a single DynamoDB table. +#[derive(Clone)] +pub struct DynamoDbProvider { + inner: Arc, +} + +struct DynamoDbProviderInner { + client: Client, + table_name: String, +} + +impl DynamoDbProvider { + pub fn new(client: Client, table_name: String) -> Self { + Self { + inner: Arc::new(DynamoDbProviderInner { client, table_name }), + } + } + + fn table(&self) -> &str { + &self.inner.table_name + } + + fn client(&self) -> &Client { + &self.inner.client + } +} + +impl ConfigProvider for DynamoDbProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + let result = self + .client() + .query() + .table_name(self.table()) + .key_condition_expression("begins_with(PK, :prefix)") + .expression_attribute_values(":prefix", AttributeValue::S("BUCKET#".into())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + let items = result.items(); + let mut buckets = Vec::with_capacity(items.len()); + + for item in items { + if let Some(json_val) = item.get("config_json") { + if let Ok(s) = json_val.as_s() { + if let Ok(config) = serde_json::from_str::(s) { + buckets.push(config); + } + } + } + } + + Ok(buckets) + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + let result = self + .client() + .get_item() + .table_name(self.table()) + .key("PK", AttributeValue::S(format!("BUCKET#{}", name))) + .key("SK", AttributeValue::S("CONFIG".into())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + match result.item() { + Some(item) => { + let json_val = item + .get("config_json") + .and_then(|v| v.as_s().ok()) + .ok_or_else(|| ProxyError::ConfigError("missing config_json".into()))?; + + let config = serde_json::from_str(json_val) + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + Ok(Some(config)) + } + None => Ok(None), + } + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + let result = self + .client() + .get_item() + .table_name(self.table()) + .key("PK", AttributeValue::S(format!("ROLE#{}", role_id))) + .key("SK", AttributeValue::S("CONFIG".into())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + match result.item() { + Some(item) => { + let json_val = item + .get("config_json") + .and_then(|v| v.as_s().ok()) + .ok_or_else(|| ProxyError::ConfigError("missing config_json".into()))?; + + let config = serde_json::from_str(json_val) + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + Ok(Some(config)) + } + None => Ok(None), + } + } + + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let result = self + .client() + .get_item() + .table_name(self.table()) + .key( + "PK", + AttributeValue::S(format!("CRED#{}", access_key_id)), + ) + .key("SK", AttributeValue::S("LONG_LIVED".into())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + match result.item() { + Some(item) => { + let json_val = item + .get("config_json") + .and_then(|v| v.as_s().ok()) + .ok_or_else(|| ProxyError::ConfigError("missing config_json".into()))?; + + let config = serde_json::from_str(json_val) + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + Ok(Some(config)) + } + None => Ok(None), + } + } + + async fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> Result<(), ProxyError> { + let json = + serde_json::to_string(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; + + // TTL for DynamoDB auto-expiry + let ttl_epoch = cred.expiration.timestamp(); + + self.client() + .put_item() + .table_name(self.table()) + .item( + "PK", + AttributeValue::S(format!("CRED#{}", cred.access_key_id)), + ) + .item("SK", AttributeValue::S("TEMPORARY".into())) + .item("config_json", AttributeValue::S(json)) + .item("ttl", AttributeValue::N(ttl_epoch.to_string())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + Ok(()) + } + + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let result = self + .client() + .get_item() + .table_name(self.table()) + .key( + "PK", + AttributeValue::S(format!("CRED#{}", access_key_id)), + ) + .key("SK", AttributeValue::S("TEMPORARY".into())) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + match result.item() { + Some(item) => { + let json_val = item + .get("config_json") + .and_then(|v| v.as_s().ok()) + .ok_or_else(|| ProxyError::ConfigError("missing config_json".into()))?; + + let cred: TemporaryCredentials = serde_json::from_str(json_val) + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + // Check expiration + if cred.expiration <= chrono::Utc::now() { + return Ok(None); + } + + Ok(Some(cred)) + } + None => Ok(None), + } + } +} diff --git a/crates/libs/core/src/config/http.rs b/crates/libs/core/src/config/http.rs new file mode 100644 index 0000000..01e9436 --- /dev/null +++ b/crates/libs/core/src/config/http.rs @@ -0,0 +1,178 @@ +//! HTTP API-backed configuration provider. +//! +//! Fetches configuration from a centralized REST API. Useful when you have +//! a control plane service that manages proxy configuration. +//! +//! # Expected API Contract +//! +//! The API should expose: +//! - `GET /buckets` → `Vec` +//! - `GET /buckets/{name}` → `Option` +//! - `GET /roles/{role_id}` → `Option` +//! - `GET /credentials/{access_key_id}` → `Option` +//! - `POST /temporary-credentials` → stores a `TemporaryCredentials` +//! - `GET /temporary-credentials/{access_key_id}` → `Option` +//! +//! # Example +//! +//! ```rust,ignore +//! use s3_proxy_core::config::http::HttpProvider; +//! +//! let provider = HttpProvider::new( +//! "https://config-api.internal:8080".to_string(), +//! Some("Bearer my-api-token".to_string()), +//! ); +//! ``` + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use std::sync::Arc; + +/// Configuration provider that reads from a REST API. +#[derive(Clone)] +pub struct HttpProvider { + inner: Arc, +} + +struct HttpProviderInner { + base_url: String, + client: reqwest::Client, + auth_header: Option, +} + +impl HttpProvider { + /// Create a new HTTP config provider. + /// + /// `base_url`: The base URL of the config API (no trailing slash). + /// `auth_header`: Optional Authorization header value (e.g., "Bearer ..."). + pub fn new(base_url: String, auth_header: Option) -> Self { + Self { + inner: Arc::new(HttpProviderInner { + base_url: base_url.trim_end_matches('/').to_string(), + client: reqwest::Client::new(), + auth_header, + }), + } + } + + fn request(&self, path: &str) -> reqwest::RequestBuilder { + let mut req = self + .inner + .client + .get(format!("{}{}", self.inner.base_url, path)); + if let Some(ref auth) = self.inner.auth_header { + req = req.header("authorization", auth); + } + req + } +} + +impl ConfigProvider for HttpProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + let resp = self + .request("/buckets") + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + resp.json() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string())) + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + let resp = self + .request(&format!("/buckets/{}", name)) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + if resp.status().as_u16() == 404 { + return Ok(None); + } + + resp.json() + .await + .map(Some) + .map_err(|e| ProxyError::ConfigError(e.to_string())) + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + let resp = self + .request(&format!("/roles/{}", role_id)) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + if resp.status().as_u16() == 404 { + return Ok(None); + } + + resp.json() + .await + .map(Some) + .map_err(|e| ProxyError::ConfigError(e.to_string())) + } + + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let resp = self + .request(&format!("/credentials/{}", access_key_id)) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + if resp.status().as_u16() == 404 { + return Ok(None); + } + + resp.json() + .await + .map(Some) + .map_err(|e| ProxyError::ConfigError(e.to_string())) + } + + async fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> Result<(), ProxyError> { + let mut req = self + .inner + .client + .post(format!("{}/temporary-credentials", self.inner.base_url)) + .json(cred); + + if let Some(ref auth) = self.inner.auth_header { + req = req.header("authorization", auth); + } + + req.send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + Ok(()) + } + + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let resp = self + .request(&format!("/temporary-credentials/{}", access_key_id)) + .send() + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + if resp.status().as_u16() == 404 { + return Ok(None); + } + + resp.json() + .await + .map(Some) + .map_err(|e| ProxyError::ConfigError(e.to_string())) + } +} diff --git a/crates/libs/core/src/config/mod.rs b/crates/libs/core/src/config/mod.rs new file mode 100644 index 0000000..d96e03c --- /dev/null +++ b/crates/libs/core/src/config/mod.rs @@ -0,0 +1,79 @@ +//! Configuration provider abstraction and implementations. +//! +//! The [`ConfigProvider`] trait defines how the proxy retrieves its +//! configuration (buckets, roles, credentials) from a backend store. +//! This allows the same core logic to work with static files, databases, +//! HTTP APIs, or any other configuration source. +//! +//! # Available Implementations +//! +//! | Provider | Feature Flag | Use Case | +//! |----------|-------------|----------| +//! | [`StaticProvider`](static_file::StaticProvider) | *(always available)* | TOML/JSON config files, baked-in config | +//! | [`HttpProvider`](http::HttpProvider) | `config-http` | Centralized config API | +//! | [`DynamoDbProvider`](dynamodb::DynamoDbProvider) | `config-dynamodb` | AWS-native deployments | +//! | [`PostgresProvider`](postgres::PostgresProvider) | `config-postgres` | Database-backed config | +//! +//! # Caching +//! +//! Wrap any provider with [`CachedProvider`](cached::CachedProvider) to add +//! in-memory TTL-based caching. This is recommended for providers that make +//! network calls (HTTP, DynamoDB, Postgres). +//! +//! ```rust,ignore +//! use s3_proxy_core::config::{cached::CachedProvider, static_file::StaticProvider}; +//! use std::time::Duration; +//! +//! let base = StaticProvider::from_file("config.toml").unwrap(); +//! let cached = CachedProvider::new(base, Duration::from_secs(300)); +//! ``` + +pub mod cached; +pub mod static_file; + +#[cfg(feature = "config-http")] +pub mod http; + +#[cfg(feature = "config-dynamodb")] +pub mod dynamodb; + +#[cfg(feature = "config-postgres")] +pub mod postgres; + +use crate::error::ProxyError; +use crate::maybe_send::{MaybeSend, MaybeSync}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use std::future::Future; + +/// Trait for retrieving proxy configuration from a backend store. +/// +/// Implementations should be cheap to clone (wrap inner state in `Arc`). +/// +/// Methods use [`MaybeSend`] bounds — on native targets this resolves to `Send` +/// (required by Tokio's task spawning), on WASM it's a no-op (allowing `!Send` +/// JS interop types). +pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { + fn list_buckets(&self) -> impl Future, ProxyError>> + MaybeSend; + + fn get_bucket(&self, name: &str) -> impl Future, ProxyError>> + MaybeSend; + + fn get_role(&self, role_id: &str) -> impl Future, ProxyError>> + MaybeSend; + + /// Look up a long-lived credential by its access key ID. + fn get_credential( + &self, + access_key_id: &str, + ) -> impl Future, ProxyError>> + MaybeSend; + + /// Store a temporary credential (minted by the STS API). + fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> impl Future> + MaybeSend; + + /// Look up a temporary credential by its access key ID. + fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> impl Future, ProxyError>> + MaybeSend; +} diff --git a/crates/libs/core/src/config/postgres.rs b/crates/libs/core/src/config/postgres.rs new file mode 100644 index 0000000..6c54977 --- /dev/null +++ b/crates/libs/core/src/config/postgres.rs @@ -0,0 +1,165 @@ +//! PostgreSQL-backed configuration provider. +//! +//! Stores configuration in a Postgres database. Good for deployments where +//! you already have a Postgres instance and want transactional config updates. +//! +//! # Required Tables +//! +//! ```sql +//! CREATE TABLE proxy_buckets ( +//! name TEXT PRIMARY KEY, +//! config_json JSONB NOT NULL +//! ); +//! +//! CREATE TABLE proxy_roles ( +//! role_id TEXT PRIMARY KEY, +//! config_json JSONB NOT NULL +//! ); +//! +//! CREATE TABLE proxy_credentials ( +//! access_key_id TEXT PRIMARY KEY, +//! credential_type TEXT NOT NULL, -- 'long_lived' or 'temporary' +//! config_json JSONB NOT NULL, +//! expires_at TIMESTAMPTZ +//! ); +//! +//! CREATE INDEX idx_credentials_expires ON proxy_credentials(expires_at) +//! WHERE credential_type = 'temporary'; +//! ``` +//! +//! # Example +//! +//! ```rust,ignore +//! use s3_proxy_core::config::postgres::PostgresProvider; +//! use sqlx::PgPool; +//! +//! let pool = PgPool::connect("postgres://user:pass@localhost/s3proxy").await?; +//! let provider = PostgresProvider::new(pool); +//! ``` + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use sqlx::PgPool; +use std::sync::Arc; + +/// Configuration provider backed by PostgreSQL. +#[derive(Clone)] +pub struct PostgresProvider { + pool: Arc, +} + +impl PostgresProvider { + pub fn new(pool: PgPool) -> Self { + Self { + pool: Arc::new(pool), + } + } +} + +impl ConfigProvider for PostgresProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + let rows: Vec<(serde_json::Value,)> = + sqlx::query_as("SELECT config_json FROM proxy_buckets") + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + rows.into_iter() + .map(|(json,)| { + serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) + }) + .collect() + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + let row: Option<(serde_json::Value,)> = + sqlx::query_as("SELECT config_json FROM proxy_buckets WHERE name = $1") + .bind(name) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + row.map(|(json,)| { + serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) + }) + .transpose() + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + let row: Option<(serde_json::Value,)> = + sqlx::query_as("SELECT config_json FROM proxy_roles WHERE role_id = $1") + .bind(role_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + row.map(|(json,)| { + serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) + }) + .transpose() + } + + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let row: Option<(serde_json::Value,)> = sqlx::query_as( + "SELECT config_json FROM proxy_credentials + WHERE access_key_id = $1 AND credential_type = 'long_lived'", + ) + .bind(access_key_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + row.map(|(json,)| { + serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) + }) + .transpose() + } + + async fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> Result<(), ProxyError> { + let json = + serde_json::to_value(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; + + sqlx::query( + "INSERT INTO proxy_credentials (access_key_id, credential_type, config_json, expires_at) + VALUES ($1, 'temporary', $2, $3) + ON CONFLICT (access_key_id) DO UPDATE + SET config_json = EXCLUDED.config_json, expires_at = EXCLUDED.expires_at", + ) + .bind(&cred.access_key_id) + .bind(&json) + .bind(cred.expiration) + .execute(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + Ok(()) + } + + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let row: Option<(serde_json::Value,)> = sqlx::query_as( + "SELECT config_json FROM proxy_credentials + WHERE access_key_id = $1 + AND credential_type = 'temporary' + AND expires_at > NOW()", + ) + .bind(access_key_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + + row.map(|(json,)| { + serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) + }) + .transpose() + } +} diff --git a/crates/libs/core/src/config/static_file.rs b/crates/libs/core/src/config/static_file.rs new file mode 100644 index 0000000..8f6e35f --- /dev/null +++ b/crates/libs/core/src/config/static_file.rs @@ -0,0 +1,165 @@ +//! Static file-based configuration provider. +//! +//! Loads configuration from a TOML or JSON file at startup. Stores temporary +//! credentials in memory. Suitable for simple deployments or development. + +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use serde::Deserialize; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +/// Full configuration file structure. +#[derive(Debug, Clone, Deserialize)] +pub struct StaticConfig { + #[serde(default)] + pub buckets: Vec, + #[serde(default)] + pub roles: Vec, + #[serde(default)] + pub credentials: Vec, +} + +/// Configuration provider backed by a static TOML/JSON file. +/// +/// # Example +/// +/// ```rust,ignore +/// let provider = StaticProvider::from_toml(r#" +/// [[buckets]] +/// name = "public-data" +/// backend_endpoint = "https://s3.amazonaws.com" +/// backend_bucket = "my-real-bucket" +/// backend_region = "us-east-1" +/// backend_access_key_id = "AKIA..." +/// backend_secret_access_key = "..." +/// anonymous_access = true +/// allowed_roles = [] +/// "#)?; +/// ``` +#[derive(Clone)] +pub struct StaticProvider { + inner: Arc, +} + +struct StaticProviderInner { + config: StaticConfig, + /// In-memory store for temporary credentials. + temp_creds: RwLock>, +} + +impl StaticProvider { + /// Parse a TOML string into a provider. + pub fn from_toml(toml_str: &str) -> Result { + let config: StaticConfig = + toml::from_str(toml_str).map_err(|e| ProxyError::ConfigError(e.to_string()))?; + Ok(Self::from_config(config)) + } + + /// Parse a JSON string into a provider. + pub fn from_json(json_str: &str) -> Result { + let config: StaticConfig = + serde_json::from_str(json_str).map_err(|e| ProxyError::ConfigError(e.to_string()))?; + Ok(Self::from_config(config)) + } + + /// Read and parse a TOML file. + pub fn from_file(path: &str) -> Result { + let content = + std::fs::read_to_string(path).map_err(|e| ProxyError::ConfigError(e.to_string()))?; + if path.ends_with(".json") { + Self::from_json(&content) + } else { + Self::from_toml(&content) + } + } + + pub fn from_config(config: StaticConfig) -> Self { + Self { + inner: Arc::new(StaticProviderInner { + config, + temp_creds: RwLock::new(HashMap::new()), + }), + } + } +} + +impl ConfigProvider for StaticProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + Ok(self.inner.config.buckets.clone()) + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + Ok(self + .inner + .config + .buckets + .iter() + .find(|b| b.name == name) + .cloned()) + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + Ok(self + .inner + .config + .roles + .iter() + .find(|r| r.role_id == role_id) + .cloned()) + } + + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + Ok(self + .inner + .config + .credentials + .iter() + .find(|c| c.access_key_id == access_key_id) + .cloned()) + } + + async fn store_temporary_credential( + &self, + cred: &TemporaryCredentials, + ) -> Result<(), ProxyError> { + let mut map = self + .inner + .temp_creds + .write() + .map_err(|e| ProxyError::Internal(e.to_string()))?; + + // Evict expired entries opportunistically + let now = chrono::Utc::now(); + map.retain(|_, v| v.expiration > now); + + map.insert(cred.access_key_id.clone(), cred.clone()); + Ok(()) + } + + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + let map = self + .inner + .temp_creds + .read() + .map_err(|e| ProxyError::Internal(e.to_string()))?; + + let cred = map.get(access_key_id).cloned(); + + // Check expiration + if let Some(ref c) = cred { + if c.expiration <= chrono::Utc::now() { + return Ok(None); + } + } + + Ok(cred) + } +} diff --git a/crates/libs/core/src/error.rs b/crates/libs/core/src/error.rs new file mode 100644 index 0000000..4e19ba7 --- /dev/null +++ b/crates/libs/core/src/error.rs @@ -0,0 +1,71 @@ +//! Error types for the proxy. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ProxyError { + #[error("bucket not found: {0}")] + BucketNotFound(String), + + #[error("access denied")] + AccessDenied, + + #[error("signature mismatch")] + SignatureDoesNotMatch, + + #[error("invalid request: {0}")] + InvalidRequest(String), + + #[error("missing authentication")] + MissingAuth, + + #[error("expired credentials")] + ExpiredCredentials, + + #[error("invalid OIDC token: {0}")] + InvalidOidcToken(String), + + #[error("role not found: {0}")] + RoleNotFound(String), + + #[error("backend error: {0}")] + BackendError(String), + + #[error("config error: {0}")] + ConfigError(String), + + #[error("internal error: {0}")] + Internal(String), +} + +impl ProxyError { + /// Return the S3-compatible XML error code. + pub fn s3_error_code(&self) -> &'static str { + match self { + Self::BucketNotFound(_) => "NoSuchBucket", + Self::AccessDenied => "AccessDenied", + Self::SignatureDoesNotMatch => "SignatureDoesNotMatch", + Self::InvalidRequest(_) => "InvalidRequest", + Self::MissingAuth => "AccessDenied", + Self::ExpiredCredentials => "ExpiredToken", + Self::InvalidOidcToken(_) => "InvalidIdentityToken", + Self::RoleNotFound(_) => "AccessDenied", + Self::BackendError(_) => "InternalError", + Self::ConfigError(_) => "InternalError", + Self::Internal(_) => "InternalError", + } + } + + /// HTTP status code for this error. + pub fn status_code(&self) -> u16 { + match self { + Self::BucketNotFound(_) => 404, + Self::AccessDenied | Self::MissingAuth | Self::ExpiredCredentials => 403, + Self::SignatureDoesNotMatch => 403, + Self::InvalidRequest(_) => 400, + Self::InvalidOidcToken(_) => 400, + Self::RoleNotFound(_) => 403, + Self::BackendError(_) | Self::ConfigError(_) | Self::Internal(_) => 500, + } + } +} diff --git a/crates/libs/core/src/lib.rs b/crates/libs/core/src/lib.rs new file mode 100644 index 0000000..39911b6 --- /dev/null +++ b/crates/libs/core/src/lib.rs @@ -0,0 +1,28 @@ +//! # s3-proxy-core +//! +//! Runtime-agnostic core library for the S3 proxy gateway. +//! +//! This crate defines the trait abstractions that allow the proxy to run on +//! multiple runtimes (Tokio/Hyper for containers, Cloudflare Workers for edge) +//! without either runtime leaking into the core logic. +//! +//! ## Key Abstractions +//! +//! - [`stream::BodyStream`] — abstract over response/request body types across runtimes +//! - [`backend::BackendClient`] — make signed outbound requests to backing object stores +//! - [`config::ConfigProvider`] — retrieve bucket/role/credential configuration from any backend +//! - [`auth`] — SigV4 request verification and credential resolution +//! - [`s3::request`] — parse incoming S3 API requests into typed operations +//! - [`s3::response`] — serialize S3 XML responses +//! - [`proxy::ProxyHandler`] — the main request handler that ties everything together + +pub mod auth; +pub mod backend; +pub mod config; +pub mod error; +pub mod maybe_send; +pub mod proxy; +pub mod resolver; +pub mod s3; +pub mod stream; +pub mod types; diff --git a/crates/libs/core/src/maybe_send.rs b/crates/libs/core/src/maybe_send.rs new file mode 100644 index 0000000..9fae035 --- /dev/null +++ b/crates/libs/core/src/maybe_send.rs @@ -0,0 +1,45 @@ +//! Conditional `Send`/`Sync` bounds for multi-runtime compatibility. +//! +//! On native targets (x86_64, aarch64, etc.), `MaybeSend` resolves to `Send` +//! and `MaybeSync` resolves to `Sync`. This satisfies Tokio's requirement +//! that spawned futures are `Send`. +//! +//! On `wasm32` targets, these traits are no-ops — blanket-implemented for all +//! types. This allows Cloudflare Workers code to use `!Send` JS interop types +//! (`Rc>`, `JsValue`, etc.) without constraint violations. +//! +//! ## Usage +//! +//! Use `MaybeSend` instead of `Send` in trait bounds throughout the core: +//! +//! ```rust,ignore +//! use s3_proxy_core::maybe_send::MaybeSend; +//! +//! pub trait MyTrait: MaybeSend { +//! fn do_work(&self) -> impl Future + MaybeSend; +//! } +//! ``` + +// --- Native targets: MaybeSend = Send, MaybeSync = Sync --- + +#[cfg(not(target_arch = "wasm32"))] +pub trait MaybeSend: Send {} +#[cfg(not(target_arch = "wasm32"))] +impl MaybeSend for T {} + +#[cfg(not(target_arch = "wasm32"))] +pub trait MaybeSync: Sync {} +#[cfg(not(target_arch = "wasm32"))] +impl MaybeSync for T {} + +// --- WASM targets: MaybeSend and MaybeSync are no-ops --- + +#[cfg(target_arch = "wasm32")] +pub trait MaybeSend {} +#[cfg(target_arch = "wasm32")] +impl MaybeSend for T {} + +#[cfg(target_arch = "wasm32")] +pub trait MaybeSync {} +#[cfg(target_arch = "wasm32")] +impl MaybeSync for T {} diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs new file mode 100644 index 0000000..c5d2577 --- /dev/null +++ b/crates/libs/core/src/proxy.rs @@ -0,0 +1,357 @@ +//! The main proxy handler that ties together resolution and backend forwarding. +//! +//! [`ProxyHandler`] is generic over the runtime's body type, backend client, +//! and request resolver. This allows it to be used identically on both +//! the server (Tokio/Hyper) and worker (Cloudflare Workers) runtimes. + +use crate::backend::{BackendClient, BackendRequest, S3RequestSigner, UNSIGNED_PAYLOAD}; +use crate::error::ProxyError; +use crate::resolver::{ResolvedAction, RequestResolver}; +use crate::s3::list_rewrite; +use crate::s3::response::ErrorResponse; +use crate::stream::BodyStream; +use crate::types::{BucketConfig, S3Operation}; +use bytes::Bytes; +use http::{HeaderMap, Method}; +use url::Url; +use uuid::Uuid; + +/// The core proxy handler, generic over runtime primitives. +/// +/// # Type Parameters +/// +/// - `C`: The backend HTTP client for outbound requests to the backing store +/// - `R`: The request resolver that decides what action to take for each request +pub struct ProxyHandler { + client: C, + resolver: R, +} + +impl ProxyHandler +where + C: BackendClient, + R: RequestResolver, +{ + pub fn new(client: C, resolver: R) -> Self { + Self { client, resolver } + } + + /// Handle an incoming S3 request. + /// + /// This is the main entry point. It: + /// 1. Resolves the request via the resolver (parse, auth, authorize) + /// 2. Forwards the request to the backing store or returns a synthetic response + /// 3. Optionally rewrites list response XML + pub async fn handle_request( + &self, + method: Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + body: C::Body, + ) -> ProxyResult { + let request_id = Uuid::new_v4().to_string(); + + tracing::info!( + request_id = %request_id, + method = %method, + path = %path, + query = ?query, + "incoming request" + ); + + match self + .handle_inner(method, path, query, headers, body) + .await + { + Ok(resp) => { + tracing::info!( + request_id = %request_id, + status = resp.status, + "request completed" + ); + resp + } + Err(err) => { + tracing::warn!( + request_id = %request_id, + error = %err, + status = err.status_code(), + s3_code = %err.s3_error_code(), + "request failed" + ); + error_response(&err, path, &request_id) + } + } + } + + async fn handle_inner( + &self, + method: Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + body: C::Body, + ) -> Result, ProxyError> { + let action = self.resolver.resolve(&method, path, query, headers).await?; + + match action { + ResolvedAction::Response { + status, + headers: resp_headers, + body: resp_body, + } => Ok(ProxyResult { + status, + headers: resp_headers, + body: C::Body::from_bytes(resp_body), + }), + ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite, + } => { + self.forward_to_backend(&method, &operation, &bucket_config, headers, body, list_rewrite.as_ref()) + .await + } + } + } + + async fn forward_to_backend( + &self, + method: &Method, + operation: &S3Operation, + bucket_config: &BucketConfig, + original_headers: &HeaderMap, + body: C::Body, + list_rewrite: Option<&crate::resolver::ListRewrite>, + ) -> Result, ProxyError> { + // Build the backend URL + let backend_url = build_backend_url(bucket_config, operation)?; + + tracing::debug!(backend_url = %backend_url, "forwarding request to backend"); + + let mut headers = HeaderMap::new(); + + // Forward relevant headers + for header_name in &[ + "content-type", + "content-length", + "content-md5", + "range", + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + ] { + if let Some(val) = original_headers.get(*header_name) { + headers.insert(*header_name, val.clone()); + } + } + + // Only sign the outbound request if the backend has credentials configured. + // Public backends (e.g. source.coop) don't need signing. + let has_credentials = !bucket_config.backend_access_key_id.is_empty() + && !bucket_config.backend_secret_access_key.is_empty(); + + let parsed_url = Url::parse(&backend_url) + .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; + + if has_credentials { + let signer = S3RequestSigner::new( + bucket_config.backend_access_key_id.clone(), + bucket_config.backend_secret_access_key.clone(), + bucket_config.backend_region.clone(), + ); + signer.sign_request(method, &parsed_url, &mut headers, UNSIGNED_PAYLOAD)?; + tracing::trace!("outbound request signed with SigV4"); + } else { + // For unsigned requests, still set the host header + let host = parsed_url + .host_str() + .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; + let host_header = if let Some(port) = parsed_url.port() { + format!("{}:{}", host, port) + } else { + host.to_string() + }; + headers.insert("host", host_header.parse().unwrap()); + tracing::trace!("outbound request unsigned (public backend)"); + } + + let backend_req = BackendRequest { + method: method.clone(), + url: backend_url, + headers, + body, + }; + + let backend_resp = self.client.send_request(backend_req).await?; + + tracing::debug!( + status = backend_resp.status, + "backend response received" + ); + + // Apply list rewrite if configured and this is a successful list response + if let Some(rewrite) = list_rewrite { + if matches!(operation, S3Operation::ListBucket { .. }) + && backend_resp.status >= 200 + && backend_resp.status < 300 + { + // List responses are small XML — safe to buffer + let body_bytes = backend_resp + .body + .read_to_bytes() + .await + .map_err(|e| ProxyError::Internal(format!("failed to read list response: {}", e)))?; + let xml_str = String::from_utf8_lossy(&body_bytes); + let rewritten = list_rewrite::rewrite_list_response(&xml_str, rewrite); + return Ok(ProxyResult { + status: backend_resp.status, + headers: backend_resp.headers, + body: C::Body::from_bytes(Bytes::from(rewritten)), + }); + } + } + + Ok(ProxyResult { + status: backend_resp.status, + headers: backend_resp.headers, + body: backend_resp.body, + }) + } +} + +/// The result of handling a proxy request. +pub struct ProxyResult { + pub status: u16, + pub headers: HeaderMap, + pub body: B, +} + +fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { + let xml = ErrorResponse::from_proxy_error(err, resource, request_id).to_xml(); + let body = B::from_bytes(Bytes::from(xml)); + let mut headers = HeaderMap::new(); + headers.insert("content-type", "application/xml".parse().unwrap()); + + ProxyResult { + status: err.status_code(), + headers, + body, + } +} + +fn build_backend_url( + config: &BucketConfig, + operation: &S3Operation, +) -> Result { + let base = config.backend_endpoint.trim_end_matches('/'); + let bucket = &config.backend_bucket; + let bucket_is_empty = bucket.is_empty(); + + let key = match operation { + S3Operation::GetObject { key, .. } + | S3Operation::HeadObject { key, .. } + | S3Operation::PutObject { key, .. } + | S3Operation::CreateMultipartUpload { key, .. } + | S3Operation::UploadPart { key, .. } + | S3Operation::CompleteMultipartUpload { key, .. } + | S3Operation::AbortMultipartUpload { key, .. } => { + let mut full_key = String::new(); + if let Some(prefix) = &config.backend_prefix { + full_key.push_str(prefix.trim_end_matches('/')); + full_key.push('/'); + } + full_key.push_str(key); + full_key + } + S3Operation::ListBucket { raw_query, .. } => { + let base_url = if bucket_is_empty { + base.to_string() + } else { + format!("{}/{}", base, bucket) + }; + let query_string = build_list_query_string(raw_query.as_deref(), config); + if query_string.is_empty() { + return Ok(base_url); + } + return Ok(format!("{}?{}", base_url, query_string)); + } + _ => return Err(ProxyError::Internal("unexpected operation".into())), + }; + + // Build URL: skip bucket segment when backend_bucket is empty (e.g. source.coop + // where the endpoint itself is the bucket root) + let mut url = if bucket_is_empty { + format!("{}/{}", base, key) + } else { + format!("{}/{}/{}", base, bucket, key) + }; + + match operation { + S3Operation::CreateMultipartUpload { .. } => { + url.push_str("?uploads"); + } + S3Operation::UploadPart { + upload_id, + part_number, + .. + } => { + url.push_str(&format!("?partNumber={}&uploadId={}", part_number, upload_id)); + } + S3Operation::CompleteMultipartUpload { upload_id, .. } + | S3Operation::AbortMultipartUpload { upload_id, .. } => { + url.push_str(&format!("?uploadId={}", upload_id)); + } + _ => {} + } + + Ok(url) +} + +/// Build the query string for a ListBucket backend request. +/// +/// - Forwards all incoming query params verbatim +/// - Prepends `backend_prefix` to the `prefix` param if configured +/// - Injects `list-type=2` if not specified (default to ListObjectsV2) +/// - Injects `max-keys=1000` if not specified +fn build_list_query_string(raw_query: Option<&str>, config: &BucketConfig) -> String { + let mut params: Vec<(String, String)> = raw_query + .map(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }) + .unwrap_or_default(); + + // Merge backend_prefix into the prefix param + if let Some(backend_prefix) = &config.backend_prefix { + let bp = backend_prefix.trim_end_matches('/'); + if !bp.is_empty() { + if let Some((_k, v)) = params.iter_mut().find(|(k, _)| k == "prefix") { + // Prepend backend_prefix to the client-supplied prefix + *v = format!("{}/{}", bp, v); + } else { + // No client prefix — set prefix to the backend_prefix (with trailing /) + // so the list is scoped to the backend_prefix directory + params.push(("prefix".to_string(), format!("{}/", bp))); + } + } + } + + // Default to ListObjectsV2 if no list-type specified + if !params.iter().any(|(k, _)| k == "list-type") { + params.push(("list-type".to_string(), "2".to_string())); + } + + // Default max-keys to 1000 if not specified + if !params.iter().any(|(k, _)| k == "max-keys") { + params.push(("max-keys".to_string(), "1000".to_string())); + } + + // Re-encode + url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(params.iter()) + .finish() +} diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs new file mode 100644 index 0000000..2d3db43 --- /dev/null +++ b/crates/libs/core/src/resolver.rs @@ -0,0 +1,194 @@ +//! Request resolution abstraction. +//! +//! The [`RequestResolver`] trait decouples "what to do with a request" from +//! the proxy handler itself. Each product (static config, Source Cooperative, +//! etc.) implements its own resolver. The proxy handler simply calls +//! `resolver.resolve()` and acts on the [`ResolvedAction`]. + +use crate::auth; +use crate::config::ConfigProvider; +use crate::error::ProxyError; +use crate::maybe_send::{MaybeSend, MaybeSync}; +use crate::s3::request::{self, HostStyle}; +use crate::s3::response::{BucketEntry, BucketList, BucketOwner, ListAllMyBucketsResult}; +use crate::types::{BucketConfig, S3Operation}; +use bytes::Bytes; +use http::{HeaderMap, Method}; +use std::future::Future; + +/// Trait for resolving an incoming request into an action the proxy should take. +/// +/// Implementations encapsulate namespace mapping, authentication, authorization, +/// and any request rewriting logic specific to a product or deployment mode. +pub trait RequestResolver: Clone + MaybeSend + MaybeSync + 'static { + fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> impl Future> + MaybeSend; +} + +/// The action the proxy handler should take after resolution. +pub enum ResolvedAction { + /// Forward the request to a backend. Core handles URL building, signing, streaming. + Proxy { + operation: S3Operation, + bucket_config: BucketConfig, + /// Optional rewrite rule for list response XML. + list_rewrite: Option, + }, + /// Return a synthetic response directly (small XML, never a stream). + Response { + status: u16, + headers: HeaderMap, + body: Bytes, + }, +} + +/// Describes how to rewrite `` and `` values in list response XML. +#[derive(Debug, Clone)] +pub struct ListRewrite { + /// Prefix to strip from the beginning of values. + pub strip_prefix: String, + /// Prefix to add after stripping. + pub add_prefix: String, +} + +/// Default resolver backed by a [`ConfigProvider`]. +/// +/// Extracts the S3 operation from the request, looks up the bucket in the +/// config, authenticates and authorizes, then returns a [`ResolvedAction::Proxy`]. +/// `ListBuckets` is handled as a synthetic [`ResolvedAction::Response`]. +#[derive(Clone)] +pub struct DefaultResolver

{ + config: P, + virtual_host_domain: Option, +} + +impl

DefaultResolver

{ + pub fn new(config: P, virtual_host_domain: Option) -> Self { + Self { + config, + virtual_host_domain, + } + } +} + +impl RequestResolver for DefaultResolver

{ + async fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> Result { + // Determine host style + let host_style = determine_host_style(headers, self.virtual_host_domain.as_deref()); + + // Parse the S3 operation + let operation = request::parse_s3_request(method, path, query, headers, host_style)?; + tracing::debug!(operation = ?operation, "parsed S3 operation"); + + // Handle STS requests separately (no bucket lookup needed) + if let S3Operation::AssumeRoleWithWebIdentity { .. } = &operation { + tracing::info!("STS AssumeRoleWithWebIdentity request"); + return Err(ProxyError::InvalidRequest( + "STS endpoint: use s3-proxy-auth crate for OIDC token exchange".into(), + )); + } + + // Handle ListBuckets — returns virtual bucket list from config, no backend call + if matches!(operation, S3Operation::ListBuckets) { + let buckets = self.config.list_buckets().await?; + tracing::info!(count = buckets.len(), "listing virtual buckets"); + let xml = ListAllMyBucketsResult { + owner: BucketOwner { + id: "s3-proxy".to_string(), + display_name: "s3-proxy".to_string(), + }, + buckets: BucketList { + buckets: buckets + .iter() + .map(|b| BucketEntry { + name: b.name.clone(), + creation_date: "2024-01-01T00:00:00.000Z".to_string(), + }) + .collect(), + }, + } + .to_xml(); + + let mut resp_headers = HeaderMap::new(); + resp_headers.insert("content-type", "application/xml".parse().unwrap()); + return Ok(ResolvedAction::Response { + status: 200, + headers: resp_headers, + body: Bytes::from(xml), + }); + } + + // Get bucket name and look up config + let bucket_name = operation_bucket(&operation) + .ok_or_else(|| ProxyError::InvalidRequest("no bucket in request".into()))?; + + let bucket_config = self + .config + .get_bucket(&bucket_name) + .await? + .ok_or_else(|| { + tracing::warn!(bucket = %bucket_name, "bucket not found in config"); + ProxyError::BucketNotFound(bucket_name.clone()) + })?; + + tracing::debug!( + bucket = %bucket_name, + backend_endpoint = %bucket_config.backend_endpoint, + "resolved bucket config" + ); + + // Authenticate + let identity = auth::resolve_identity(headers, &self.config).await?; + tracing::debug!(identity = ?identity, "resolved identity"); + + // Authorize + auth::authorize(&identity, &operation, &bucket_config)?; + tracing::trace!("authorization passed"); + + Ok(ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite: None, + }) + } +} + +fn operation_bucket(op: &S3Operation) -> Option { + match op { + S3Operation::GetObject { bucket, .. } + | S3Operation::HeadObject { bucket, .. } + | S3Operation::PutObject { bucket, .. } + | S3Operation::ListBucket { bucket, .. } + | S3Operation::CreateMultipartUpload { bucket, .. } + | S3Operation::UploadPart { bucket, .. } + | S3Operation::CompleteMultipartUpload { bucket, .. } + | S3Operation::AbortMultipartUpload { bucket, .. } => Some(bucket.clone()), + S3Operation::ListBuckets => None, + S3Operation::AssumeRoleWithWebIdentity { .. } => None, + } +} + +fn determine_host_style(headers: &HeaderMap, virtual_host_domain: Option<&str>) -> HostStyle { + if let Some(domain) = virtual_host_domain { + if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) { + let host = host.split(':').next().unwrap_or(host); + if let Some(bucket) = host.strip_suffix(&format!(".{}", domain)) { + return HostStyle::VirtualHosted { + bucket: bucket.to_string(), + }; + } + } + } + HostStyle::Path +} diff --git a/crates/libs/core/src/s3/list_rewrite.rs b/crates/libs/core/src/s3/list_rewrite.rs new file mode 100644 index 0000000..aedbb6a --- /dev/null +++ b/crates/libs/core/src/s3/list_rewrite.rs @@ -0,0 +1,112 @@ +//! XML rewriting for S3 list responses. +//! +//! When a backend prefix is configured, the backend returns keys that include +//! the prefix. This module strips that prefix and optionally prepends a new +//! one, so clients see the expected key structure. + +use crate::resolver::ListRewrite; + +/// Rewrite `` and `` element values in a ListObjectsV2 XML response +/// according to the given [`ListRewrite`] rule. +pub fn rewrite_list_response(xml: &str, rewrite: &ListRewrite) -> String { + let mut result = xml.to_string(); + result = rewrite_xml_element_values(&result, "Key", &rewrite.strip_prefix, &rewrite.add_prefix); + result = rewrite_xml_element_values(&result, "Prefix", &rewrite.strip_prefix, &rewrite.add_prefix); + result +} + +/// Replace prefix in XML element values: +/// `old_prefix/rest` -> `new_prefix/rest` +fn rewrite_xml_element_values( + xml: &str, + tag: &str, + old_prefix: &str, + new_prefix: &str, +) -> String { + let open = format!("<{}>", tag); + let close = format!("", tag); + let mut result = String::with_capacity(xml.len()); + let mut remaining = xml; + + while let Some(start_idx) = remaining.find(&open) { + result.push_str(&remaining[..start_idx + open.len()]); + remaining = &remaining[start_idx + open.len()..]; + + if let Some(end_idx) = remaining.find(&close) { + let value = &remaining[..end_idx]; + if let Some(stripped) = value.strip_prefix(old_prefix) { + if new_prefix.is_empty() { + result.push_str(stripped.trim_start_matches('/')); + } else { + result.push_str(new_prefix); + if !stripped.is_empty() && !stripped.starts_with('/') { + result.push('/'); + } + result.push_str(stripped.trim_start_matches('/')); + } + } else { + result.push_str(value); + } + result.push_str(&close); + remaining = &remaining[end_idx + close.len()..]; + } else { + // Malformed XML — just append the rest + break; + } + } + result.push_str(remaining); + result +} + +/// Extract the text content of the first occurrence of `...`. +pub fn extract_xml_element<'a>(xml: &'a str, tag: &str) -> Option<&'a str> { + let open = format!("<{}>", tag); + let close = format!("", tag); + let start = xml.find(&open)? + open.len(); + let end = xml[start..].find(&close)? + start; + Some(&xml[start..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rewrite_strips_prefix() { + let xml = r#"base/mirror/file.csv"#; + let rewrite = ListRewrite { + strip_prefix: "base/mirror/".to_string(), + add_prefix: "repo".to_string(), + }; + let result = rewrite_list_response(xml, &rewrite); + assert!(result.contains("repo/file.csv"), "got: {}", result); + } + + #[test] + fn test_rewrite_strip_only() { + let xml = r#"prefix/file.csv"#; + let rewrite = ListRewrite { + strip_prefix: "prefix/".to_string(), + add_prefix: String::new(), + }; + let result = rewrite_list_response(xml, &rewrite); + assert!(result.contains("file.csv")); + } + + #[test] + fn test_extract_xml_element() { + let xml = r#"some/prefix/"#; + assert_eq!(extract_xml_element(xml, "Prefix"), Some("some/prefix/")); + } + + #[test] + fn test_no_match_preserves_xml() { + let xml = r#"other/file.csv"#; + let rewrite = ListRewrite { + strip_prefix: "nonexistent/".to_string(), + add_prefix: "new/".to_string(), + }; + let result = rewrite_list_response(xml, &rewrite); + assert!(result.contains("other/file.csv")); + } +} diff --git a/crates/libs/core/src/s3/mod.rs b/crates/libs/core/src/s3/mod.rs new file mode 100644 index 0000000..6f27e96 --- /dev/null +++ b/crates/libs/core/src/s3/mod.rs @@ -0,0 +1,3 @@ +pub mod list_rewrite; +pub mod request; +pub mod response; diff --git a/crates/libs/core/src/s3/request.rs b/crates/libs/core/src/s3/request.rs new file mode 100644 index 0000000..22833a2 --- /dev/null +++ b/crates/libs/core/src/s3/request.rs @@ -0,0 +1,189 @@ +//! Parse incoming HTTP requests into typed S3 operations. + +use crate::error::ProxyError; +use crate::types::S3Operation; +use http::Method; + +/// Extract the bucket and key from a path-style S3 request. +/// +/// Path-style: `/{bucket}/{key}` +/// Virtual-hosted-style: Host header `{bucket}.s3.example.com` with path `/{key}` +pub fn parse_s3_request( + method: &Method, + uri_path: &str, + query: Option<&str>, + _headers: &http::HeaderMap, + host_style: HostStyle, +) -> Result { + // Check for STS actions in query params + if let Some(q) = query { + let params: Vec<(String, String)> = url::form_urlencoded::parse(q.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let action = params.iter().find(|(k, _)| k == "Action"); + if let Some((_, action_value)) = action { + if action_value == "AssumeRoleWithWebIdentity" { + return parse_sts_request(¶ms); + } + } + } + + // GET / with path-style → ListBuckets (no bucket in path) + if matches!(host_style, HostStyle::Path) && uri_path.trim_start_matches('/').is_empty() { + if *method == Method::GET { + return Ok(S3Operation::ListBuckets); + } + return Err(ProxyError::InvalidRequest("unsupported operation on /".into())); + } + + let (bucket, key) = match host_style { + HostStyle::Path => parse_path_style(uri_path)?, + HostStyle::VirtualHosted { bucket } => (bucket, uri_path.trim_start_matches('/').to_string()), + }; + + build_s3_operation(method, bucket, key, query) +} + +/// Build an [`S3Operation`] from an already-extracted bucket, key, and query. +/// +/// This is used by both [`parse_s3_request`] and custom resolvers that parse +/// the path themselves (e.g., Source Cooperative). +pub fn build_s3_operation( + method: &Method, + bucket: String, + key: String, + query: Option<&str>, +) -> Result { + let query_params = parse_query_params(query); + + // Check for multipart upload query params + let upload_id = query_params + .iter() + .find(|(k, _)| k == "uploadId") + .map(|(_, v)| v.clone()); + + let has_uploads = query_params.iter().any(|(k, _)| k == "uploads"); + + match method { + &Method::GET => { + if key.is_empty() { + // ListBucket — pass the raw query string through so the proxy + // can forward all list params (prefix, delimiter, max-keys, + // continuation-token, list-type, start-after, etc.) to the backend. + Ok(S3Operation::ListBucket { + bucket, + raw_query: query.map(|q| q.to_string()), + }) + } else { + Ok(S3Operation::GetObject { bucket, key }) + } + } + &Method::HEAD => Ok(S3Operation::HeadObject { bucket, key }), + &Method::PUT => { + if let Some(upload_id) = upload_id { + let part_number = query_params + .iter() + .find(|(k, _)| k == "partNumber") + .and_then(|(_, v)| v.parse().ok()) + .ok_or_else(|| ProxyError::InvalidRequest("missing partNumber".into()))?; + + Ok(S3Operation::UploadPart { + bucket, + key, + upload_id, + part_number, + }) + } else { + Ok(S3Operation::PutObject { bucket, key }) + } + } + &Method::POST => { + if has_uploads { + Ok(S3Operation::CreateMultipartUpload { bucket, key }) + } else if let Some(upload_id) = upload_id { + Ok(S3Operation::CompleteMultipartUpload { + bucket, + key, + upload_id, + }) + } else { + Err(ProxyError::InvalidRequest( + "unsupported POST operation".into(), + )) + } + } + &Method::DELETE => { + if let Some(upload_id) = upload_id { + Ok(S3Operation::AbortMultipartUpload { + bucket, + key, + upload_id, + }) + } else { + Err(ProxyError::InvalidRequest( + "unsupported DELETE operation".into(), + )) + } + } + _ => Err(ProxyError::InvalidRequest(format!( + "unsupported method: {}", + method + ))), + } +} + +#[derive(Debug, Clone)] +pub enum HostStyle { + /// Path-style: `/{bucket}/{key}` + Path, + /// Virtual-hosted-style: bucket extracted from Host header. + VirtualHosted { bucket: String }, +} + +fn parse_path_style(path: &str) -> Result<(String, String), ProxyError> { + let trimmed = path.trim_start_matches('/'); + if trimmed.is_empty() { + return Err(ProxyError::InvalidRequest("empty path".into())); + } + + match trimmed.split_once('/') { + Some((bucket, key)) => Ok((bucket.to_string(), key.to_string())), + None => Ok((trimmed.to_string(), String::new())), + } +} + +fn parse_query_params(query: Option<&str>) -> Vec<(String, String)> { + query + .map(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }) + .unwrap_or_default() +} + +fn parse_sts_request(params: &[(String, String)]) -> Result { + let role_arn = params + .iter() + .find(|(k, _)| k == "RoleArn") + .map(|(_, v)| v.clone()) + .ok_or_else(|| ProxyError::InvalidRequest("missing RoleArn".into()))?; + + let web_identity_token = params + .iter() + .find(|(k, _)| k == "WebIdentityToken") + .map(|(_, v)| v.clone()) + .ok_or_else(|| ProxyError::InvalidRequest("missing WebIdentityToken".into()))?; + + let duration_seconds = params + .iter() + .find(|(k, _)| k == "DurationSeconds") + .and_then(|(_, v)| v.parse().ok()); + + Ok(S3Operation::AssumeRoleWithWebIdentity { + role_arn, + web_identity_token, + duration_seconds, + }) +} diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs new file mode 100644 index 0000000..f73391b --- /dev/null +++ b/crates/libs/core/src/s3/response.rs @@ -0,0 +1,184 @@ +//! S3 XML response serialization. + +use quick_xml::se::to_string as xml_to_string; +use serde::Serialize; + +use crate::error::ProxyError; + +/// S3 Error response XML. +#[derive(Debug, Serialize)] +#[serde(rename = "Error")] +pub struct ErrorResponse { + #[serde(rename = "Code")] + pub code: String, + #[serde(rename = "Message")] + pub message: String, + #[serde(rename = "Resource")] + pub resource: String, + #[serde(rename = "RequestId")] + pub request_id: String, +} + +impl ErrorResponse { + pub fn from_proxy_error(err: &ProxyError, resource: &str, request_id: &str) -> Self { + Self { + code: err.s3_error_code().to_string(), + message: err.to_string(), + resource: resource.to_string(), + request_id: request_id.to_string(), + } + } + + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_else(|_| "InternalError".to_string()) + ) + } +} + +/// InitiateMultipartUpload response. +#[derive(Debug, Serialize)] +#[serde(rename = "InitiateMultipartUploadResult")] +pub struct InitiateMultipartUploadResult { + #[serde(rename = "Bucket")] + pub bucket: String, + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "UploadId")] + pub upload_id: String, +} + +impl InitiateMultipartUploadResult { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} + +/// CompleteMultipartUpload response. +#[derive(Debug, Serialize)] +#[serde(rename = "CompleteMultipartUploadResult")] +pub struct CompleteMultipartUploadResult { + #[serde(rename = "Location")] + pub location: String, + #[serde(rename = "Bucket")] + pub bucket: String, + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "ETag")] + pub etag: String, +} + +impl CompleteMultipartUploadResult { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} + +/// Request body for CompleteMultipartUpload. +#[derive(Debug, serde::Deserialize)] +#[serde(rename = "CompleteMultipartUpload")] +pub struct CompleteMultipartUploadRequest { + #[serde(rename = "Part")] + pub parts: Vec, +} + +#[derive(Debug, serde::Deserialize)] +pub struct CompletePart { + #[serde(rename = "PartNumber")] + pub part_number: u32, + #[serde(rename = "ETag")] + pub etag: String, +} + +/// ListAllMyBucketsResult response (for `GET /`). +#[derive(Debug, Serialize)] +#[serde(rename = "ListAllMyBucketsResult")] +pub struct ListAllMyBucketsResult { + #[serde(rename = "Owner")] + pub owner: BucketOwner, + #[serde(rename = "Buckets")] + pub buckets: BucketList, +} + +#[derive(Debug, Serialize)] +pub struct BucketOwner { + #[serde(rename = "ID")] + pub id: String, + #[serde(rename = "DisplayName")] + pub display_name: String, +} + +#[derive(Debug, Serialize)] +pub struct BucketList { + #[serde(rename = "Bucket")] + pub buckets: Vec, +} + +#[derive(Debug, Serialize)] +pub struct BucketEntry { + #[serde(rename = "Name")] + pub name: String, + #[serde(rename = "CreationDate")] + pub creation_date: String, +} + +impl ListAllMyBucketsResult { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} + +/// STS AssumeRoleWithWebIdentity response. +#[derive(Debug, Serialize)] +#[serde(rename = "AssumeRoleWithWebIdentityResponse")] +pub struct AssumeRoleWithWebIdentityResponse { + #[serde(rename = "AssumeRoleWithWebIdentityResult")] + pub result: AssumeRoleWithWebIdentityResult, +} + +#[derive(Debug, Serialize)] +pub struct AssumeRoleWithWebIdentityResult { + #[serde(rename = "Credentials")] + pub credentials: StsCredentials, + #[serde(rename = "AssumedRoleUser")] + pub assumed_role_user: AssumedRoleUser, +} + +#[derive(Debug, Serialize)] +pub struct StsCredentials { + #[serde(rename = "AccessKeyId")] + pub access_key_id: String, + #[serde(rename = "SecretAccessKey")] + pub secret_access_key: String, + #[serde(rename = "SessionToken")] + pub session_token: String, + #[serde(rename = "Expiration")] + pub expiration: String, +} + +#[derive(Debug, Serialize)] +pub struct AssumedRoleUser { + #[serde(rename = "AssumedRoleId")] + pub assumed_role_id: String, + #[serde(rename = "Arn")] + pub arn: String, +} + +impl AssumeRoleWithWebIdentityResponse { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} diff --git a/crates/libs/core/src/stream.rs b/crates/libs/core/src/stream.rs new file mode 100644 index 0000000..54b2e86 --- /dev/null +++ b/crates/libs/core/src/stream.rs @@ -0,0 +1,40 @@ +//! Stream abstraction for runtime-agnostic body handling. +//! +//! The key insight: the core proxy logic almost never needs to inspect or +//! transform the bytes flowing through it. For GET/PUT, the body is opaque — +//! it comes in from one side and goes out the other. This means our trait +//! can be minimal: we just need to know a body type exists and can be passed +//! around. +//! +//! Each runtime provides its own concrete type: +//! - Server runtime: `hyper::body::Incoming` / `http_body_util::Full` +//! - Worker runtime: a wrapper around JS `ReadableStream` +//! +//! The only time the core reads body bytes is for `CompleteMultipartUpload` +//! (parsing the XML manifest), which uses the `read_to_bytes` method. + +use bytes::Bytes; +use std::future::Future; + +use crate::maybe_send::MaybeSend; + +/// Trait representing a streaming body type. +/// +/// This is intentionally minimal. The core passes bodies through opaquely; +/// it never iterates over chunks except when it must parse a small request body. +pub trait BodyStream: Sized + MaybeSend + 'static { + type Error: std::error::Error + Send + Sync + 'static; + + /// Consume the body and collect all bytes. + /// Used only for small bodies like XML manifests, never for large object data. + fn read_to_bytes(self) -> impl Future> + MaybeSend; + + /// Create a body from raw bytes. + fn from_bytes(bytes: Bytes) -> Self; + + /// Create an empty body. + fn empty() -> Self; + + /// Content length, if known. + fn content_length(&self) -> Option; +} diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs new file mode 100644 index 0000000..692bc97 --- /dev/null +++ b/crates/libs/core/src/types.rs @@ -0,0 +1,173 @@ +//! Shared types used across the proxy. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// Configuration for a virtual bucket exposed by the proxy. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BucketConfig { + /// The virtual bucket name exposed to clients. + pub name: String, + + /// The backing object store endpoint (e.g., "https://s3.amazonaws.com"). + pub backend_endpoint: String, + + /// The real bucket name on the backing store. + pub backend_bucket: String, + + /// Optional prefix to prepend to all keys when forwarding. + pub backend_prefix: Option, + + /// The region to use when signing requests to the backend. + pub backend_region: String, + + /// Credentials for signing outbound requests to the backing store. + pub backend_access_key_id: String, + pub backend_secret_access_key: String, + + /// Whether this bucket allows anonymous (unsigned) access. + pub anonymous_access: bool, + + /// IAM role ARNs that are allowed to access this bucket. + /// Empty means only anonymous access (if enabled) or long-lived credentials. + pub allowed_roles: Vec, +} + +/// Configuration for an IAM role that can be assumed via STS. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoleConfig { + /// The role identifier (used as the RoleArn in AssumeRoleWithWebIdentity). + pub role_id: String, + + /// Human-readable name. + pub name: String, + + /// OIDC provider URLs trusted by this role (e.g., "https://token.actions.githubusercontent.com"). + pub trusted_oidc_issuers: Vec, + + /// Required audience claim value. + pub required_audience: Option, + + /// Conditions on the subject claim (glob patterns). + /// e.g., "repo:myorg/myrepo:ref:refs/heads/main" + pub subject_conditions: Vec, + + /// Buckets and prefixes this role can access. + pub allowed_scopes: Vec, + + /// Maximum session duration in seconds. + pub max_session_duration_secs: u64, +} + +/// Defines what a credential is allowed to access. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessScope { + pub bucket: String, + /// Allowed key prefixes. Empty means full bucket access. + pub prefixes: Vec, + /// Allowed actions. + pub actions: Vec, +} + +/// S3 actions that can be authorized. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Action { + GetObject, + HeadObject, + PutObject, + ListBucket, + CreateMultipartUpload, + UploadPart, + CompleteMultipartUpload, + AbortMultipartUpload, +} + +/// A long-lived access credential stored in the config backend. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredCredential { + pub access_key_id: String, + /// This is the HMAC signing key, not stored in plaintext ideally. + pub secret_access_key: String, + pub principal_name: String, + pub allowed_scopes: Vec, + pub created_at: DateTime, + pub expires_at: Option>, + pub enabled: bool, +} + +/// Temporary credentials minted by the STS API. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TemporaryCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: DateTime, + pub allowed_scopes: Vec, + pub assumed_role_id: String, + pub source_identity: String, +} + +/// Represents the resolved identity after authentication. +#[derive(Debug, Clone)] +pub enum ResolvedIdentity { + Anonymous, + LongLived { + credential: StoredCredential, + }, + Temporary { + credentials: TemporaryCredentials, + }, +} + +/// The parsed S3 operation extracted from an incoming request. +#[derive(Debug, Clone)] +pub enum S3Operation { + GetObject { + bucket: String, + key: String, + }, + HeadObject { + bucket: String, + key: String, + }, + PutObject { + bucket: String, + key: String, + }, + CreateMultipartUpload { + bucket: String, + key: String, + }, + UploadPart { + bucket: String, + key: String, + upload_id: String, + part_number: u32, + }, + CompleteMultipartUpload { + bucket: String, + key: String, + upload_id: String, + }, + AbortMultipartUpload { + bucket: String, + key: String, + upload_id: String, + }, + ListBucket { + bucket: String, + /// Raw query string from the incoming request, forwarded to the backend. + /// The proxy may modify `prefix` (prepend backend_prefix) and inject + /// defaults for `max-keys` and `list-type`. + raw_query: Option, + }, + /// List all virtual buckets exposed by the proxy. + ListBuckets, + /// STS AssumeRoleWithWebIdentity (served on the same endpoint). + AssumeRoleWithWebIdentity { + role_arn: String, + web_identity_token: String, + duration_seconds: Option, + }, +} diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml new file mode 100644 index 0000000..a616673 --- /dev/null +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "s3-proxy-cf-workers" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Cloudflare Workers runtime for the S3 proxy gateway" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +s3-proxy-core = { workspace = true, features = [] } +s3-proxy-auth.workspace = true +bytes.workspace = true +http.workspace = true +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +thiserror.workspace = true +chrono.workspace = true +quick-xml.workspace = true +url.workspace = true + +# Cloudflare Workers SDK +worker = "0.7" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +js-sys = "0.3" +web-sys = { version = "0.3", features = [ + "Headers", + "ReadableStream", + "Request", + "RequestInit", + "Response", + "ResponseInit", +] } +console_error_panic_hook = "0.1.7" +getrandom = { version = "0.2", features = ["js"] } diff --git a/crates/runtimes/cf-workers/README.md b/crates/runtimes/cf-workers/README.md new file mode 100644 index 0000000..35ed42e --- /dev/null +++ b/crates/runtimes/cf-workers/README.md @@ -0,0 +1,143 @@ +# s3-proxy-cf-workers + +Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, with JS `ReadableStream` passthrough — response bytes from the backing store never touch WASM memory. + +## How It Works + +``` +Client request + -> Worker fetch handler (lib.rs) + -> Convert worker::Request -> http types + -> Pick resolver: + - SOURCE_API_URL set? -> SourceCoopResolver (dynamic Source Cooperative backends) + - Otherwise -> DefaultResolver (static PROXY_CONFIG) + -> ProxyHandler::handle_request() (from s3-proxy-core) + -> WorkerBackendClient sends fetch() to backend + -> Response returned to client (ReadableStream passthrough) +``` + +The `WorkerBackendClient` uses the Workers Fetch API for outbound requests. For GET responses, the JS `ReadableStream` from the backend is passed directly to the outbound `Response` — bytes never cross the WASM boundary. + +## Module Overview + +``` +src/ +├── lib.rs Worker entry point, request/response conversion (thin adapter) +├── body.rs WorkerBody implementing BodyStream (Bytes | ReadableStream | Empty) +├── client.rs WorkerBackendClient implementing BackendClient via Fetch API +├── source_api.rs HTTP client for the Source Cooperative API +└── source_resolver.rs SourceCoopResolver implementing RequestResolver +``` + +## Operating Modes + +### Static Config Mode (default) + +Reads bucket configuration from the `PROXY_CONFIG` environment variable. Uses `DefaultResolver` which handles standard S3 path/virtual-host parsing, SigV4 authentication, and scope-based authorization. + +```toml +# wrangler.toml +[vars] +PROXY_CONFIG = '{"buckets":[...],"roles":[...],"credentials":[...]}' +VIRTUAL_HOST_DOMAIN = "s3.example.com" # optional, for virtual-hosted style +``` + +### Source Cooperative Mode + +When `SOURCE_API_URL` is set, the worker uses `SourceCoopResolver` which resolves backends dynamically from the Source Cooperative API. This resolver implements a custom URL namespace: + +- `GET /` — synthetic empty ListBuckets +- `GET /{account_id}` — lists repositories via Source API, returns synthetic ListObjectsV2 with CommonPrefixes +- `GET /{account_id}?prefix=repo_id/subdir/` — proxies to the repo's backend with prefix rewriting +- `GET|PUT|... /{account_id}/{repo_id}/{key}` — proxies to the repo's S3 backend + +Authentication is handled by the Source API permissions endpoint rather than the core auth module. + +```toml +# wrangler.toml +[vars] +SOURCE_API_URL = "https://api.source.coop" + +# Set via wrangler secret: +# wrangler secret put SOURCE_API_KEY +``` + +### Implementing a Custom Resolver + +To add a new operating mode, implement `RequestResolver` in a new module: + +```rust +use s3_proxy_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; +use s3_proxy_core::error::ProxyError; + +#[derive(Clone)] +struct MyResolver { /* ... */ } + +impl RequestResolver for MyResolver { + async fn resolve( + &self, + method: &http::Method, + path: &str, + query: Option<&str>, + headers: &http::HeaderMap, + ) -> Result { + // Parse the URL, authenticate, resolve a BucketConfig, + // and return ResolvedAction::Proxy or ResolvedAction::Response. + todo!() + } +} +``` + +Then add a branch in `lib.rs`: + +```rust +if let Ok(my_config) = env.var("MY_MODE") { + let resolver = MyResolver::new(/* ... */); + let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + let result = handler.handle_request(method, &path, query.as_deref(), &headers, body).await; + return build_worker_response(result); +} +``` + +## Local Development + +Run MinIO via Docker Compose from the repo root, then start the worker with Wrangler: + +```bash +# Terminal 1: start MinIO (from repo root) +docker compose up + +# Terminal 2: start the worker dev server +cd crates/runtimes/cf-workers +npx wrangler dev +``` + +Wrangler starts a local server (default `:8787`). The `wrangler.toml` includes a `PROXY_CONFIG` var pointing at `localhost:9000` (MinIO). + +```bash +# Test it +curl http://localhost:8787/public-data/hello.txt +``` + +Note: `wrangler dev` runs the WASM module in a local Workerd runtime. Outbound `fetch()` calls from the worker to `localhost:9000` work because Wrangler's dev server runs on the host network. + +## Deployment + +```bash +cd crates/runtimes/cf-workers + +# Build and deploy to Cloudflare +npx wrangler deploy +``` + +For production, update the `PROXY_CONFIG` var in `wrangler.toml` (or set it via the Cloudflare dashboard / `wrangler secret`) to point at your real backend endpoints. + +## Why a Separate Crate + +Cloudflare Workers compile to `wasm32-unknown-unknown` and link against `worker-rs`, `wasm-bindgen`, and `web-sys`. These dependencies are incompatible with native targets. Keeping them isolated means `cargo build` for the server crate doesn't pull in WASM tooling, and `wrangler build` for this crate doesn't pull in Tokio. + +This crate must always be built with `--target wasm32-unknown-unknown`: + +```bash +cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +``` diff --git a/crates/runtimes/cf-workers/node_modules/.cache/wrangler/wrangler-account.json b/crates/runtimes/cf-workers/node_modules/.cache/wrangler/wrangler-account.json new file mode 100644 index 0000000..f879448 --- /dev/null +++ b/crates/runtimes/cf-workers/node_modules/.cache/wrangler/wrangler-account.json @@ -0,0 +1,6 @@ +{ + "account": { + "id": "83a892ddc9739cbda441192079db2a3d", + "name": "alukach" + } +} \ No newline at end of file diff --git a/crates/runtimes/cf-workers/node_modules/.mf/cf.json b/crates/runtimes/cf-workers/node_modules/.mf/cf.json new file mode 100644 index 0000000..a5b75fb --- /dev/null +++ b/crates/runtimes/cf-workers/node_modules/.mf/cf.json @@ -0,0 +1 @@ +{"httpProtocol":"HTTP/1.1","clientAcceptEncoding":"gzip, deflate, br","requestPriority":"","edgeRequestKeepAliveStatus":1,"requestHeaderNames":{},"clientTcpRtt":213,"colo":"YYZ","asn":852,"asOrganization":"TELUS Communications Inc.","country":"CA","isEUCountry":false,"city":"Toronto","continent":"NA","region":"Ontario","regionCode":"ON","timezone":"America/Toronto","longitude":"-79.39864","latitude":"43.70643","postalCode":"M5A","tlsVersion":"TLSv1.3","tlsCipher":"AEAD-AES256-GCM-SHA384","tlsClientRandom":"BlauG9Yq3y4A4iUSUbDmmqu1JGI5+OLuye+ersJL+gc=","tlsClientCiphersSha1":"JZtiTn8H/ntxORk+XXvU2EvNoz8=","tlsClientExtensionsSha1":"Y7DIC8A6G0/aXviZ8ie/xDbJb7g=","tlsClientExtensionsSha1Le":"6e+q3vPm88rSgMTN/h7WTTxQ2wQ=","tlsExportedAuthenticator":{"clientHandshake":"3064b89d755921a2a20c9672b8aef197269b146c504f5d63b8567a56ccf74c0a1303617b2a56d4011208c545cc2afa27","serverHandshake":"eeb8136f336b11f22f286b20527040da29ffbd5e4fdf4f69ceb6c496734c9b1f1c8ec4fe395b4cc47c6512f6d48018b7","clientFinished":"ea40955f5323ec638c6a7cd82dd4bb3aee3e18c32612b802604e79eb9614f58041235484cd134fa697d1e6808437d17f","serverFinished":"1fff715305f8ef3fdd5dd95e84530dc2af4f86bb85d7236e192cfa0bf3f6861d56bba039114e7c9d4329fdf1ddd2d6c8"},"tlsClientHelloLength":"386","tlsClientAuth":{"certPresented":"0","certVerified":"NONE","certRevoked":"0","certIssuerDN":"","certSubjectDN":"","certIssuerDNRFC2253":"","certSubjectDNRFC2253":"","certIssuerDNLegacy":"","certSubjectDNLegacy":"","certSerial":"","certIssuerSerial":"","certSKI":"","certIssuerSKI":"","certFingerprintSHA1":"","certFingerprintSHA256":"","certNotBefore":"","certNotAfter":""},"verifiedBotCategory":"","botManagement":{"corporateProxy":false,"verifiedBot":false,"jsDetection":{"passed":false},"staticResource":false,"detectionIds":{},"score":99}} \ No newline at end of file diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs new file mode 100644 index 0000000..93b8d10 --- /dev/null +++ b/crates/runtimes/cf-workers/src/body.rs @@ -0,0 +1,122 @@ +//! Worker body type implementing `BodyStream`. +//! +//! The key optimization: response bodies from the backend Fetch API are +//! `ReadableStream` objects in JS. Rather than reading them into Rust memory, +//! we pass them through opaquely. The stream only touches Rust when the core +//! needs to parse a small body (e.g., CompleteMultipartUpload XML manifest). + +use bytes::Bytes; +use js_sys::Uint8Array; +use s3_proxy_core::stream::BodyStream; +use wasm_bindgen_futures::JsFuture; + +/// Body type for the Cloudflare Workers runtime. +/// +/// Most request/response bodies flow through as opaque JS `ReadableStream` +/// objects, never touching Rust memory. The `Bytes` variant is used only +/// for small bodies constructed in Rust (error responses, XML manifests). +pub enum WorkerBody { + /// Raw bytes (small bodies constructed in Rust). + Bytes(Bytes), + /// A JS ReadableStream passed through opaquely. + Stream(web_sys::ReadableStream), + /// No body. + Empty, +} + +#[derive(Debug, thiserror::Error)] +#[error("worker body error: {0}")] +pub struct WorkerBodyError(pub String); + +impl BodyStream for WorkerBody { + type Error = WorkerBodyError; + + async fn read_to_bytes(self) -> Result { + match self { + WorkerBody::Bytes(b) => Ok(b), + WorkerBody::Empty => Ok(Bytes::new()), + WorkerBody::Stream(stream) => { + // Consume the ReadableStream into bytes. + // This is only called for small bodies (XML manifests), never for + // large object data. + read_stream_to_bytes(stream).await + } + } + } + + fn from_bytes(bytes: Bytes) -> Self { + if bytes.is_empty() { + WorkerBody::Empty + } else { + WorkerBody::Bytes(bytes) + } + } + + fn empty() -> Self { + WorkerBody::Empty + } + + fn content_length(&self) -> Option { + match self { + WorkerBody::Bytes(b) => Some(b.len() as u64), + WorkerBody::Empty => Some(0), + // Stream length is unknown — the backend will set Content-Length + // in the response headers if applicable. + WorkerBody::Stream(_) => None, + } + } +} + +impl WorkerBody { + /// Convert to a JsValue for use as a Fetch API request body. + /// Returns `None` for empty bodies (Fetch API interprets absent body as no body). + pub fn into_js_body(self) -> Option { + match self { + WorkerBody::Empty => None, + WorkerBody::Bytes(b) => { + let uint8 = Uint8Array::from(b.as_ref()); + Some(uint8.into()) + } + WorkerBody::Stream(stream) => Some(stream.into()), + } + } + + /// Create a `WorkerBody` from a `web_sys::Response` by extracting its + /// ReadableStream body without consuming it into bytes. + pub fn from_ws_response(response: &web_sys::Response) -> Self { + match response.body() { + Some(stream) => WorkerBody::Stream(stream), + None => WorkerBody::Empty, + } + } + + /// Create a `WorkerBody` from a `web_sys::Request` by extracting its + /// ReadableStream body. + pub fn from_ws_request(request: &web_sys::Request) -> Self { + match request.body() { + Some(stream) => WorkerBody::Stream(stream), + None => WorkerBody::Empty, + } + } +} + +/// Read a JS ReadableStream to completion, collecting all chunks into `Bytes`. +/// +/// Uses the JS `Response` constructor trick: `new Response(stream).arrayBuffer()` +/// which is the most efficient way to consume a stream in Workers. +async fn read_stream_to_bytes(stream: web_sys::ReadableStream) -> Result { + // Create a Response from the stream, then read its arrayBuffer + let response = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) + .map_err(|e| WorkerBodyError(format!("failed to wrap stream in Response: {:?}", e)))?; + + let array_buffer_promise = response + .array_buffer() + .map_err(|e| WorkerBodyError(format!("failed to get arrayBuffer: {:?}", e)))?; + + let array_buffer = JsFuture::from(array_buffer_promise) + .await + .map_err(|e| WorkerBodyError(format!("failed to read arrayBuffer: {:?}", e)))?; + + let uint8 = Uint8Array::new(&array_buffer); + Ok(Bytes::from(uint8.to_vec())) +} diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs new file mode 100644 index 0000000..a66807e --- /dev/null +++ b/crates/runtimes/cf-workers/src/client.rs @@ -0,0 +1,199 @@ +//! Backend client using the Cloudflare Workers Fetch API. +//! +//! Uses `worker::Fetch` for subrequests. Response bodies for proxied S3 +//! requests are extracted as `ReadableStream` via `web_sys` for zero-copy +//! passthrough — bytes never enter Rust memory. + +use crate::body::WorkerBody; +use s3_proxy_core::backend::{BackendClient, BackendRequest, BackendResponse}; +use s3_proxy_core::error::ProxyError; +use serde::de::DeserializeOwned; +use worker::{Cache, Fetch}; + +/// Options for Cache API caching. +pub(crate) struct CacheOptions { + pub cache_ttl: u32, + pub cache_key: Option, +} + +/// Build the cache key URL for the Cache API. +/// +/// When a custom key is provided, it is formatted as `https://cache.internal/{key}` +/// because the Cache API requires valid URLs as keys. +fn cache_key_url(url: &str, opts: &CacheOptions) -> String { + match &opts.cache_key { + Some(key) => format!("https://cache.internal/{}", key), + None => url.to_string(), + } +} + +/// Fetch a URL and deserialize the JSON response body. +/// +/// Used by `source_api` for server-to-server calls to the Source Cooperative API. +/// When `cache` is provided, responses are cached using the Cloudflare Workers +/// Cache API with explicit `cache.get()` / `cache.put()` calls. +pub(crate) async fn fetch_json( + url: &str, + headers: &[(&str, &str)], + cache: Option<&CacheOptions>, +) -> Result { + // Check cache for a hit before making the request. + let cache_state = if let Some(opts) = cache { + let key = cache_key_url(url, opts); + let cf_cache = Cache::default(); + match cf_cache.get(&key, false).await { + Ok(Some(mut cached)) => { + if let Ok(text) = cached.text().await { + if let Ok(value) = serde_json::from_str(&text) { + return Ok(value); + } + } + // Cache hit but couldn't deserialize — fall through to fetch. + Some((cf_cache, key)) + } + _ => Some((cf_cache, key)), + } + } else { + None + }; + + // Build and execute the fetch request. + let mut req_init = worker::RequestInit::new(); + let worker_headers = worker::Headers::new(); + for (k, v) in headers { + worker_headers + .set(k, v) + .map_err(|e| ProxyError::Internal(format!("failed to set header: {}", e)))?; + } + req_init.with_headers(worker_headers); + + let req = worker::Request::new_with_init(url, &req_init) + .map_err(|e| ProxyError::Internal(format!("failed to create request: {}", e)))?; + + let mut resp = Fetch::Request(req) + .send() + .await + .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; + + let status = resp.status_code(); + + let text = resp + .text() + .await + .map_err(|e| ProxyError::Internal(format!("failed to read text: {}", e)))?; + + if status < 200 || status >= 300 { + return Err(ProxyError::BackendError(format!( + "API request to {} returned status {}", + url, status + ))); + } + + // Cache successful responses via the Cache API. + if let Some((cf_cache, key)) = cache_state { + let ttl = cache.unwrap().cache_ttl; + if let Ok(mut response) = worker::Response::ok(&text) { + let _ = response + .headers_mut() + .set("Cache-Control", &format!("max-age={}", ttl)); + // cache.put is fire-and-forget; ignore errors. + let _ = cf_cache.put(&key, response).await; + } + } + + serde_json::from_str(&text) + .map_err(|e| ProxyError::Internal(format!("failed to deserialize response: {}", e))) +} + +/// Backend client that uses the Workers Fetch API. +/// +/// Response bodies remain as opaque JS ReadableStreams — bytes never touch +/// Rust memory for passthrough requests (GET, PUT, etc.). +pub struct WorkerBackendClient; + +impl BackendClient for WorkerBackendClient { + type Body = WorkerBody; + + async fn send_request( + &self, + request: BackendRequest, + ) -> Result, ProxyError> { + tracing::debug!( + method = %request.method, + url = %request.url, + "worker: sending backend request via Fetch API" + ); + + // Build web_sys::Headers directly + let ws_headers = web_sys::Headers::new() + .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; + + for (key, value) in request.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } + } + + // Build web_sys::RequestInit — we use web_sys types here because + // WorkerBody needs to pass JS ReadableStream/Uint8Array as the body. + let init = web_sys::RequestInit::new(); + init.set_method(request.method.as_str()); + init.set_headers(&ws_headers.into()); + + // Set body for methods that carry one. + // Pass streams and bytes through as JS values — no materialization. + if matches!(request.method, http::Method::PUT | http::Method::POST) { + if let Some(js_body) = request.body.into_js_body() { + init.set_body(&js_body); + } + } + + let ws_request = + web_sys::Request::new_with_str_and_init(&request.url, &init).map_err(|e| { + tracing::error!(error = ?e, "failed to create web_sys::Request"); + ProxyError::BackendError(format!("failed to create request: {:?}", e)) + })?; + + // Convert to worker::Request and fetch via worker::Fetch. + let worker_req: worker::Request = ws_request.into(); + let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { + tracing::error!(url = %request.url, error = %e, "fetch to backend failed"); + ProxyError::BackendError(format!("fetch failed: {}", e)) + })?; + + let status = worker_resp.status_code(); + tracing::debug!(status = status, "worker: backend response received"); + + // Convert back to web_sys::Response to extract ReadableStream body. + let ws_response: web_sys::Response = worker_resp.into(); + + // Convert response headers + let mut resp_headers = http::HeaderMap::new(); + let response_headers = ws_response.headers(); + for name in &[ + "content-type", + "content-length", + "etag", + "last-modified", + "x-amz-request-id", + "x-amz-version-id", + "accept-ranges", + "content-range", + ] { + if let Ok(Some(value)) = response_headers.get(name) { + if let Ok(parsed) = value.parse() { + resp_headers.insert(*name, parsed); + } + } + } + + // Extract response body as a ReadableStream — zero-copy passthrough. + let body = WorkerBody::from_ws_response(&ws_response); + + Ok(BackendResponse { + status, + headers: resp_headers, + body, + }) + } +} diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs new file mode 100644 index 0000000..7ab0b7c --- /dev/null +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -0,0 +1,232 @@ +//! Cloudflare Workers runtime for the S3 proxy gateway. +//! +//! This crate provides implementations of core traits using Cloudflare Workers +//! primitives. The key advantage: response bodies from backend object stores +//! remain as JS `ReadableStream` objects throughout the proxy pipeline, avoiding +//! any conversion to/from Rust byte streams. +//! +//! # Architecture +//! +//! ```text +//! Client -> Worker (JS Request) +//! -> resolve request (core resolver or Source Cooperative resolver) +//! -> fetch from backend (JS Fetch API -> JS Response with ReadableStream body) +//! -> return JS Response with ReadableStream body directly +//! ``` +//! +//! The body bytes never touch Rust memory for GET requests. This is the primary +//! performance advantage of the multi-runtime architecture. +//! +//! # Configuration +//! +//! On Workers, configuration is loaded from: +//! - Environment variables / secrets for simple setups +//! - Workers KV for dynamic configuration +//! - The HTTP config provider for centralized config APIs +//! - **Source Cooperative API** when `SOURCE_API_URL` is set + +mod body; +mod client; +mod source_api; +mod source_resolver; +mod tracing_layer; + +use body::WorkerBody; +use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; +use s3_proxy_core::proxy::ProxyHandler; +use s3_proxy_core::resolver::DefaultResolver; +use s3_proxy_core::stream::BodyStream; +use worker::*; + +/// The Worker entry point. +/// +/// Wrangler config (`wrangler.toml`) should bind: +/// - `CONFIG` environment variable or KV namespace for configuration +/// - `VIRTUAL_HOST_DOMAIN` environment variable (optional) +/// - `SOURCE_API_URL` + `SOURCE_API_KEY` for Source Cooperative API mode +#[event(fetch)] +async fn main(req: Request, env: Env, _ctx: Context) -> Result { + // Initialize panic hook for better error messages + console_error_panic_hook::set_once(); + + // Initialize tracing subscriber (idempotent — ignored if already set) + tracing::subscriber::set_global_default(tracing_layer::WorkerSubscriber::new()).ok(); + + let method = convert_method(&req); + let url = req.url()?; + let path = url.path().to_string(); + let query = url.query().map(|q| q.to_string()); + let headers = convert_headers(&req); + + // Extract the request body as a JS ReadableStream — zero CPU cost. + let body = if matches!(method, http::Method::PUT | http::Method::POST) { + WorkerBody::from_ws_request(req.inner()) + } else { + WorkerBody::empty() + }; + + // Source Cooperative API mode: when SOURCE_API_URL is set, resolve backends + // dynamically from the Source API instead of static PROXY_CONFIG. + if let Ok(source_api_url) = env.var("SOURCE_API_URL") { + let source_api_key = env + .var("SOURCE_API_KEY") + .map(|v| v.to_string()) + .map_err(|e| { + worker::Error::RustError(format!( + "SOURCE_API_KEY required when SOURCE_API_URL is set: {}", + e + )) + })?; + + let mut cache_ttls = source_api::CacheTtls::default(); + if let Ok(v) = env.var("SOURCE_CACHE_TTL_PRODUCT") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.product = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_DATA_CONNECTION") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.data_connection = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_PERMISSIONS") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.permissions = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_ACCOUNT") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.account = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_API_KEY") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.api_key = n; + } + } + + let api_client = + source_api::SourceApiClient::new(source_api_url.to_string(), source_api_key, cache_ttls); + let resolver = source_resolver::SourceCoopResolver::new(api_client); + let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + + let result = handler + .handle_request(method, &path, query.as_deref(), &headers, body) + .await; + + return build_worker_response(result); + } + + // Load PROXY_CONFIG from environment. + // Supports two formats: + // - JSON string (e.g., set via `wrangler secret` or a plain string var) + // - JS object (e.g., set via `[vars.PROXY_CONFIG]` table in wrangler.toml) + let config = if let Ok(var) = env.var("PROXY_CONFIG") { + let config_str = var.to_string(); + tracing::debug!(config_len = config_str.len(), "loaded PROXY_CONFIG as string"); + StaticProvider::from_json(&config_str) + .map_err(|e| worker::Error::RustError(format!("config error: {}", e)))? + } else { + tracing::debug!("loading PROXY_CONFIG as object"); + let static_config: StaticConfig = env + .object_var("PROXY_CONFIG") + .map_err(|e| worker::Error::RustError(format!("config error: {}", e)))?; + StaticProvider::from_config(static_config) + }; + + let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); + let resolver = DefaultResolver::new(config, virtual_host_domain); + let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + + let result = handler + .handle_request(method, &path, query.as_deref(), &headers, body) + .await; + + build_worker_response(result) +} + +// ── Shared helpers ────────────────────────────────────────────────── + +/// Build a `worker::Response` from a `ProxyResult`, preserving stream bodies. +fn build_worker_response( + result: s3_proxy_core::proxy::ProxyResult, +) -> Result { + match result.body { + WorkerBody::Stream(stream) => { + let ws_headers = web_sys::Headers::new() + .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } + } + + let init = web_sys::ResponseInit::new(); + init.set_status(result.status); + init.set_headers(&ws_headers.into()); + + let ws_response = + web_sys::Response::new_with_opt_readable_stream_and_init(Some(&stream), &init) + .map_err(|e| { + worker::Error::RustError(format!("failed to build response: {:?}", e)) + })?; + + Ok(ws_response.into()) + } + WorkerBody::Bytes(b) => { + let resp_headers = Headers::new(); + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = resp_headers.set(key.as_str(), v); + } + } + Ok(Response::from_bytes(b.to_vec())? + .with_status(result.status) + .with_headers(resp_headers)) + } + WorkerBody::Empty => { + let resp_headers = Headers::new(); + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = resp_headers.set(key.as_str(), v); + } + } + Ok(Response::from_bytes(vec![])? + .with_status(result.status) + .with_headers(resp_headers)) + } + } +} + +fn convert_method(req: &Request) -> http::Method { + match req.method() { + Method::Get => http::Method::GET, + Method::Head => http::Method::HEAD, + Method::Post => http::Method::POST, + Method::Put => http::Method::PUT, + Method::Delete => http::Method::DELETE, + _ => http::Method::GET, + } +} + +fn convert_headers(req: &Request) -> http::HeaderMap { + let mut headers = http::HeaderMap::new(); + for name in &[ + "authorization", + "host", + "x-amz-date", + "x-amz-content-sha256", + "x-amz-security-token", + "content-type", + "content-length", + "content-md5", + "range", + ] { + if let Ok(Some(value)) = req.headers().get(name) { + if let Ok(parsed) = value.parse() { + headers.insert(*name, parsed); + } + } + } + headers +} diff --git a/crates/runtimes/cf-workers/src/source_api.rs b/crates/runtimes/cf-workers/src/source_api.rs new file mode 100644 index 0000000..8ddc5e4 --- /dev/null +++ b/crates/runtimes/cf-workers/src/source_api.rs @@ -0,0 +1,227 @@ +//! HTTP client for the Source Cooperative API. +//! +//! Makes server-to-server calls to resolve products, data connections, +//! API keys, and permissions. All calls use the Workers Fetch API via +//! the shared `fetch_json` helper in `client.rs`. + +use crate::client::{fetch_json, CacheOptions}; +use s3_proxy_core::error::ProxyError; +use serde::Deserialize; +use std::collections::HashMap; + +/// Per-endpoint cache TTLs (seconds). Set to 0 to disable caching. +pub struct CacheTtls { + pub product: u32, + pub data_connection: u32, + pub permissions: u32, + pub account: u32, + pub api_key: u32, +} + +impl Default for CacheTtls { + fn default() -> Self { + Self { + product: 5 * 60, + data_connection: 30 * 60, + permissions: 60, + account: 5 * 60, + api_key: 60, + } + } +} + +/// Client for the Source Cooperative API. +#[derive(Clone)] +pub struct SourceApiClient { + api_url: String, + api_key: String, + product_cache_ttl: u32, + data_connection_cache_ttl: u32, + permissions_cache_ttl: u32, + account_cache_ttl: u32, + api_key_cache_ttl: u32, +} + +// ── API response types ────────────────────────────────────────────── + +#[derive(Debug, Deserialize)] +pub struct SourceProduct { + pub disabled: bool, + pub data_mode: String, + pub metadata: ProductMetadata, +} + +#[derive(Debug, Deserialize)] +pub struct ProductMetadata { + pub primary_mirror: String, + pub mirrors: HashMap, +} + +#[derive(Debug, Deserialize)] +pub struct ProductMirror { + pub connection_id: String, + pub prefix: String, +} + +#[derive(Debug, Deserialize)] +pub struct DataConnection { + pub details: ConnectionDetails, + pub authentication: ConnectionAuth, +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +pub struct ConnectionDetails { + pub provider: String, + pub region: Option, + pub base_prefix: Option, + pub bucket: Option, +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +pub struct ConnectionAuth { + pub auth_type: String, + pub access_key_id: Option, + pub secret_access_key: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SourceApiKey { + pub access_key_id: String, + pub secret_access_key: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] +pub enum RepositoryPermission { + Read, + Write, +} + +/// API response for the permissions endpoint. +#[derive(Debug, Deserialize)] +pub struct PermissionsResponse { + #[serde(default)] + pub read: bool, + #[serde(default)] + pub write: bool, +} + +/// API response for account listing — contains repositories. +#[derive(Debug, Deserialize)] +pub struct AccountResponse { + #[serde(default)] + pub repositories: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct AccountRepository { + pub repository_id: String, +} + +// ── Client implementation ─────────────────────────────────────────── + +impl SourceApiClient { + pub fn new(api_url: String, api_key: String, cache_ttls: CacheTtls) -> Self { + Self { + api_url, + api_key, + product_cache_ttl: cache_ttls.product, + data_connection_cache_ttl: cache_ttls.data_connection, + permissions_cache_ttl: cache_ttls.permissions, + account_cache_ttl: cache_ttls.account, + api_key_cache_ttl: cache_ttls.api_key, + } + } + + fn auth_headers(&self) -> Vec<(&str, String)> { + vec![("authorization", self.api_key.clone())] + } + + /// `GET /api/v1/products/{account_id}/{repo_id}` + pub async fn get_product( + &self, + account_id: &str, + repo_id: &str, + ) -> Result { + let url = format!( + "{}/api/v1/products/{}/{}", + self.api_url, account_id, repo_id + ); + let auth = self.auth_headers(); + let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let cache = (self.product_cache_ttl > 0).then(|| CacheOptions { + cache_ttl: self.product_cache_ttl, + cache_key: None, + }); + fetch_json(&url, &headers, cache.as_ref()).await + } + + /// `GET /api/v1/data-connections/{id}` + pub async fn get_data_connection(&self, id: &str) -> Result { + let url = format!("{}/api/v1/data-connections/{}", self.api_url, id); + let auth = self.auth_headers(); + let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let cache = (self.data_connection_cache_ttl > 0).then(|| CacheOptions { + cache_ttl: self.data_connection_cache_ttl, + cache_key: None, + }); + fetch_json(&url, &headers, cache.as_ref()).await + } + + /// `GET /api/v1/api-keys/{access_key_id}/auth` + pub async fn get_api_key(&self, access_key_id: &str) -> Result { + let url = format!("{}/api/v1/api-keys/{}/auth", self.api_url, access_key_id); + let auth = self.auth_headers(); + let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let cache = (self.api_key_cache_ttl > 0).then(|| CacheOptions { + cache_ttl: self.api_key_cache_ttl, + cache_key: None, + }); + fetch_json(&url, &headers, cache.as_ref()).await + } + + /// `GET /api/v1/products/{account_id}/{repo_id}/permissions` + /// + /// Uses the *user's* API key (not the server key) to check permissions + /// for the authenticated user. + /// + /// A custom `cacheKey` incorporating the user's API key prevents + /// cross-user cache poisoning (the URL is the same for all users). + pub async fn get_permissions( + &self, + account_id: &str, + repo_id: &str, + user_api_key: &str, + ) -> Result { + let url = format!( + "{}/api/v1/products/{}/{}/permissions", + self.api_url, account_id, repo_id + ); + let headers = [("authorization", user_api_key)]; + let cache = (self.permissions_cache_ttl > 0).then(|| CacheOptions { + cache_ttl: self.permissions_cache_ttl, + cache_key: Some(format!( + "source-perms:{}:{}:{}", + account_id, repo_id, user_api_key + )), + }); + fetch_json(&url, &headers, cache.as_ref()).await + } + + /// `GET /api/v1/accounts/{account_id}` + pub async fn list_account_repos( + &self, + account_id: &str, + ) -> Result { + let url = format!("{}/api/v1/accounts/{}", self.api_url, account_id); + let auth = self.auth_headers(); + let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let cache = (self.account_cache_ttl > 0).then(|| CacheOptions { + cache_ttl: self.account_cache_ttl, + cache_key: None, + }); + fetch_json(&url, &headers, cache.as_ref()).await + } +} diff --git a/crates/runtimes/cf-workers/src/source_resolver.rs b/crates/runtimes/cf-workers/src/source_resolver.rs new file mode 100644 index 0000000..73f0e28 --- /dev/null +++ b/crates/runtimes/cf-workers/src/source_resolver.rs @@ -0,0 +1,390 @@ +//! [`RequestResolver`] implementation for Source Cooperative. +//! +//! Consolidates all Source Cooperative business logic (URL namespace mapping, +//! external API auth, query/response prefix rewriting, synthetic XML responses) +//! into a single resolver that the thin CF Workers adapter calls. + +use crate::source_api::SourceApiClient; +use bytes::Bytes; +use http::{HeaderMap, Method}; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::resolver::{ListRewrite, RequestResolver, ResolvedAction}; +use s3_proxy_core::s3::request::build_s3_operation; +use s3_proxy_core::types::BucketConfig; + +/// Request resolver for Source Cooperative. +/// +/// Routes requests based on the URL namespace: +/// - `/` -> synthetic empty ListBuckets +/// - `/{account_id}` -> synthetic account listing or list-with-prefix +/// - `/{account_id}/{repo_id}[/{key}]` -> proxy to backend +#[derive(Clone)] +pub struct SourceCoopResolver { + api_client: SourceApiClient, +} + +impl SourceCoopResolver { + pub fn new(api_client: SourceApiClient) -> Self { + Self { api_client } + } + + /// Resolve a bucket config from the Source API for a given account/repo. + async fn resolve_bucket_config( + &self, + account_id: &str, + repo_id: &str, + ) -> Result { + let bucket_name = format!("{}/{}", account_id, repo_id); + + let product = self + .api_client + .get_product(account_id, repo_id) + .await + .map_err(|e| { + tracing::warn!( + account_id = account_id, + repo_id = repo_id, + error = %e, + "failed to fetch product from Source API" + ); + ProxyError::BucketNotFound(bucket_name.clone()) + })?; + + if product.disabled { + return Err(ProxyError::BucketNotFound(bucket_name)); + } + + let mirror = product + .metadata + .mirrors + .get(&product.metadata.primary_mirror) + .ok_or_else(|| { + ProxyError::ConfigError(format!( + "primary mirror '{}' not found in product mirrors", + product.metadata.primary_mirror + )) + })?; + + let conn = self + .api_client + .get_data_connection(&mirror.connection_id) + .await?; + + let region = conn.details.region.unwrap_or_else(|| "us-east-1".into()); + let bucket = conn.details.bucket.unwrap_or_default(); + let base_prefix = conn.details.base_prefix.unwrap_or_default(); + + let backend_prefix = { + let bp = base_prefix.trim_end_matches('/'); + let mp = mirror.prefix.trim_end_matches('/'); + if bp.is_empty() && mp.is_empty() { + None + } else if bp.is_empty() { + Some(mp.to_string()) + } else if mp.is_empty() { + Some(bp.to_string()) + } else { + Some(format!("{}/{}", bp, mp)) + } + }; + + let endpoint = format!("https://s3.{}.amazonaws.com", region); + + Ok(BucketConfig { + name: bucket_name, + backend_endpoint: endpoint, + backend_bucket: bucket, + backend_prefix, + backend_region: region, + backend_access_key_id: conn.authentication.access_key_id.unwrap_or_default(), + backend_secret_access_key: conn.authentication.secret_access_key.unwrap_or_default(), + anonymous_access: product.data_mode == "open", + allowed_roles: vec![], + }) + } + + /// Check permissions for an authenticated user via the Source API. + async fn check_permissions( + &self, + headers: &HeaderMap, + account_id: &str, + repo_id: &str, + method: &Method, + ) -> Result<(), ProxyError> { + let auth_header = match headers.get("authorization").and_then(|v| v.to_str().ok()) { + Some(h) => h, + None => return Ok(()), // Anonymous — skip permission check + }; + + let sig = s3_proxy_core::auth::parse_sigv4_auth(auth_header)?; + + let perms = self + .api_client + .get_permissions(account_id, repo_id, &sig.access_key_id) + .await?; + + let is_write = matches!( + *method, + Method::PUT | Method::POST | Method::DELETE + ); + + if is_write && !perms.write { + tracing::warn!( + account_id = account_id, + repo_id = repo_id, + access_key_id = %sig.access_key_id, + "write permission denied by Source API" + ); + return Err(ProxyError::AccessDenied); + } + + if !is_write && !perms.read { + tracing::warn!( + account_id = account_id, + repo_id = repo_id, + access_key_id = %sig.access_key_id, + "read permission denied by Source API" + ); + return Err(ProxyError::AccessDenied); + } + + Ok(()) + } + + /// Handle `GET /{account_id}` — synthetic account listing. + async fn handle_account_listing( + &self, + account_id: &str, + ) -> Result { + let account = self + .api_client + .list_account_repos(account_id) + .await?; + + let prefixes: Vec = account + .repositories + .iter() + .map(|r| format!("{}/", r.repository_id)) + .collect(); + + let xml = synthetic_list_objects_v2_xml(account_id, &prefixes); + let mut headers = HeaderMap::new(); + headers.insert("content-type", "application/xml".parse().unwrap()); + + Ok(ResolvedAction::Response { + status: 200, + headers, + body: Bytes::from(xml), + }) + } + + /// Handle a list-with-prefix request where the prefix includes a repo_id. + /// + /// `GET /{account_id}?prefix=repo_id/subdir/...` + async fn handle_list_with_prefix( + &self, + method: &Method, + headers: &HeaderMap, + query: &str, + account_id: &str, + repo_id: &str, + ) -> Result { + let bucket_name = format!("{}/{}", account_id, repo_id); + + // Permission check + self.check_permissions(headers, account_id, repo_id, method) + .await?; + + // Rewrite query: strip `{repo_id}/` from the prefix param + let new_query = rewrite_list_prefix(query, repo_id); + + let bucket_config = self.resolve_bucket_config(account_id, repo_id).await?; + + let operation = build_s3_operation(method, bucket_name, String::new(), Some(&new_query))?; + + // Build list rewrite: strip backend prefix, add repo_id + let list_rewrite = build_list_rewrite(&bucket_config, repo_id); + + Ok(ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite, + }) + } +} + +impl RequestResolver for SourceCoopResolver { + async fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> Result { + let trimmed = path.trim_start_matches('/'); + let segments: Vec<&str> = trimmed.splitn(3, '/').collect(); + + match segments.as_slice() { + // Root: GET / -> empty ListBuckets + [] | [""] => { + let xml = empty_list_buckets_xml(); + let mut resp_headers = HeaderMap::new(); + resp_headers.insert("content-type", "application/xml".parse().unwrap()); + Ok(ResolvedAction::Response { + status: 200, + headers: resp_headers, + body: Bytes::from(xml), + }) + } + + // /{account_id} — either account listing or list-with-prefix + [account_id] if !account_id.is_empty() => { + // Check if there's a prefix query param that starts with a repo_id + if let Some(q) = query { + if let Some(prefix) = extract_query_param(q, "prefix") { + if let Some((repo_part, _rest)) = prefix.split_once('/') { + if !repo_part.is_empty() { + return self + .handle_list_with_prefix( + method, headers, q, account_id, repo_part, + ) + .await; + } + } + } + } + + // No prefix or prefix doesn't contain repo -> synthetic account listing + self.handle_account_listing(account_id).await + } + + // /{account_id}/{repo_id} or /{account_id}/{repo_id}/{key...} + [account_id, repo_id_and_rest @ ..] if !account_id.is_empty() => { + let (repo_id, key) = if repo_id_and_rest.len() == 1 { + (repo_id_and_rest[0], "") + } else { + (repo_id_and_rest[0], repo_id_and_rest[1]) + }; + + if repo_id.is_empty() { + return Err(ProxyError::InvalidRequest("empty repo_id".into())); + } + + let bucket_name = format!("{}/{}", account_id, repo_id); + + // Permission check via Source API + self.check_permissions(headers, account_id, repo_id, method) + .await?; + + // Resolve bucket config + let bucket_config = self.resolve_bucket_config(account_id, repo_id).await?; + + // Build the S3 operation + let operation = + build_s3_operation(method, bucket_name, key.to_string(), query)?; + + // For list operations, apply list rewrite + let list_rewrite = if key.is_empty() { + build_list_rewrite(&bucket_config, repo_id) + } else { + None + }; + + Ok(ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite, + }) + } + + _ => Err(ProxyError::InvalidRequest("invalid path".into())), + } + } +} + +// ── Helpers ───────────────────────────────────────────────────────── + +/// Build a [`ListRewrite`] that strips the backend prefix and prepends `repo_id`. +fn build_list_rewrite(bucket_config: &BucketConfig, repo_id: &str) -> Option { + // The backend prefix is what core prepends to list queries. We need to + // strip it from response keys and add `repo_id/` so clients see the + // expected namespace. + let strip = bucket_config + .backend_prefix + .as_deref() + .unwrap_or("") + .trim_end_matches('/'); + + if strip.is_empty() && repo_id.is_empty() { + return None; + } + + Some(ListRewrite { + strip_prefix: if strip.is_empty() { + String::new() + } else { + format!("{}/", strip) + }, + add_prefix: repo_id.to_string(), + }) +} + +fn empty_list_buckets_xml() -> String { + r#" + + source-coopsource-coop + +"# + .to_string() +} + +fn synthetic_list_objects_v2_xml(bucket: &str, common_prefixes: &[String]) -> String { + let mut xml = format!( + r#" + + {} + + {} + 1000 + false"#, + bucket, + common_prefixes.len() + ); + + for prefix in common_prefixes { + xml.push_str(&format!( + "\n {}", + prefix + )); + } + + xml.push_str("\n"); + xml +} + +fn extract_query_param(query: &str, name: &str) -> Option { + url::form_urlencoded::parse(query.as_bytes()) + .find(|(k, _)| k == name) + .map(|(_, v)| v.to_string()) +} + +fn rewrite_list_prefix(query: &str, repo_id: &str) -> String { + let params: Vec<(String, String)> = url::form_urlencoded::parse(query.as_bytes()) + .map(|(k, v)| { + if k == "prefix" { + let prefix_to_strip = format!("{}/", repo_id); + let new_v = v + .strip_prefix(&prefix_to_strip) + .unwrap_or(&v) + .to_string(); + (k.to_string(), new_v) + } else { + (k.to_string(), v.to_string()) + } + }) + .collect(); + + url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(params.iter()) + .finish() +} diff --git a/crates/runtimes/cf-workers/src/tracing_layer.rs b/crates/runtimes/cf-workers/src/tracing_layer.rs new file mode 100644 index 0000000..b241177 --- /dev/null +++ b/crates/runtimes/cf-workers/src/tracing_layer.rs @@ -0,0 +1,117 @@ +//! A lightweight `tracing` layer that routes log output to the Workers +//! `console.log` / `console.error` / `console.warn` APIs. +//! +//! This avoids pulling in `tracing-subscriber` (which depends on `std::time` +//! and other things unavailable on `wasm32`). Instead we implement a minimal +//! [`tracing::Subscriber`] that formats events and forwards them through +//! `worker::console_log!` / `console_error!` / `console_warn!`. + +use tracing::field::{Field, Visit}; +use tracing::{Event, Level, Metadata, Subscriber}; +use tracing::span; + +/// A minimal tracing subscriber that logs to the Workers console. +/// +/// Install once at the start of each request: +/// ```rust,ignore +/// tracing::subscriber::set_global_default(WorkerSubscriber::new()) +/// .ok(); // ignore if already set +/// ``` +pub struct WorkerSubscriber { + max_level: Level, +} + +impl WorkerSubscriber { + pub fn new() -> Self { + Self { + max_level: Level::DEBUG, + } + } + + pub fn with_max_level(mut self, level: Level) -> Self { + self.max_level = level; + self + } +} + +impl Subscriber for WorkerSubscriber { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.level() <= &self.max_level + } + + fn new_span(&self, _attrs: &span::Attributes<'_>) -> span::Id { + // We don't track spans — just log events. + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + fn event(&self, event: &Event<'_>) { + let metadata = event.metadata(); + let level = metadata.level(); + let target = metadata.target(); + + let mut visitor = MessageVisitor::default(); + event.record(&mut visitor); + + let msg = if visitor.fields.is_empty() { + format!("[{level}] {target}: {}", visitor.message) + } else { + format!( + "[{level}] {target}: {} {{ {} }}", + visitor.message, + visitor.fields.join(", ") + ) + }; + + // Route to the appropriate console method based on level. + // worker::console_log! and friends are macros that call into JS. + match *level { + Level::ERROR => worker::console_error!("{}", msg), + Level::WARN => worker::console_warn!("{}", msg), + _ => worker::console_log!("{}", msg), + } + } + + fn enter(&self, _span: &span::Id) {} + fn exit(&self, _span: &span::Id) {} +} + +/// Visitor that extracts the `message` field and collects remaining fields +/// into `key=value` pairs. +#[derive(Default)] +struct MessageVisitor { + message: String, + fields: Vec, +} + +impl Visit for MessageVisitor { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + self.message = format!("{:?}", value); + } else { + self.fields.push(format!("{}={:?}", field.name(), value)); + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "message" { + self.message = value.to_string(); + } else { + self.fields + .push(format!("{}=\"{}\"", field.name(), value)); + } + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.fields.push(format!("{}={}", field.name(), value)); + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.fields.push(format!("{}={}", field.name(), value)); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.fields.push(format!("{}={}", field.name(), value)); + } +} diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml new file mode 100644 index 0000000..028b975 --- /dev/null +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -0,0 +1,80 @@ +name = "s3-proxy-cf-workers" +main = "build/worker/shim.mjs" +compatibility_date = "2024-09-23" + +[build] +command = "cargo install worker-build && worker-build --release" + +[vars] +VIRTUAL_HOST_DOMAIN = "s3.local" + +# For production, consider storing this in Workers KV or a Secrets binding. +[vars.PROXY_CONFIG] +roles = [] + +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = true +backend_access_key_id = "minioadmin" +backend_bucket = "public-data" +backend_endpoint = "http://localhost:9000" +backend_region = "us-east-1" +backend_secret_access_key = "minioadmin" +name = "public-data" + +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = false +backend_access_key_id = "minioadmin" +backend_bucket = "private-uploads" +backend_endpoint = "http://localhost:9000" +backend_region = "us-east-1" +backend_secret_access_key = "minioadmin" +name = "private-uploads" + +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = true +backend_access_key_id = "" +backend_bucket = "us-west-2.opendata.source.coop" +backend_endpoint = "https://s3.us-west-2.amazonaws.com" +backend_prefix = "cholmes/" +backend_region = "us-west-2" +backend_secret_access_key = "" +name = "cholmes" + +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = true +backend_access_key_id = "" +backend_bucket = "us-west-2.opendata.source.coop" +backend_endpoint = "https://s3.us-west-2.amazonaws.com" +backend_prefix = "harvard-lil/" +backend_region = "us-west-2" +backend_secret_access_key = "" +name = "harvard-lil" + +[[vars.PROXY_CONFIG.credentials]] +access_key_id = "AKLOCAL0000000000001" +created_at = "2024-01-01T00:00:00Z" +enabled = true +principal_name = "dev-user" +secret_access_key = "localdev/secret/key/00000000000000000000" + +[[vars.PROXY_CONFIG.credentials.allowed_scopes]] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +bucket = "public-data" +prefixes = [] + +[[vars.PROXY_CONFIG.credentials.allowed_scopes]] +actions = [ + "get_object", + "head_object", + "put_object", + "list_bucket", + "create_multipart_upload", + "upload_part", + "complete_multipart_upload", +] +bucket = "private-uploads" +prefixes = [] diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml new file mode 100644 index 0000000..9b49a37 --- /dev/null +++ b/crates/runtimes/server/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "s3-proxy-server" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Tokio/Hyper runtime for the S3 proxy gateway" + +[dependencies] +s3-proxy-core = { workspace = true, features = [] } +s3-proxy-auth.workspace = true +tokio.workspace = true +hyper.workspace = true +hyper-util.workspace = true +http-body-util.workspace = true +http.workspace = true +bytes.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +serde.workspace = true +toml.workspace = true +reqwest = { workspace = true, features = ["stream"] } +thiserror.workspace = true diff --git a/crates/runtimes/server/README.md b/crates/runtimes/server/README.md new file mode 100644 index 0000000..e311d21 --- /dev/null +++ b/crates/runtimes/server/README.md @@ -0,0 +1,115 @@ +# s3-proxy-server + +Tokio/Hyper runtime for the S3 proxy gateway. This is the container-deployment crate — it wires the core library into a production HTTP server using native Rust async I/O. + +## What This Crate Provides + +Three concrete implementations of core traits, plus a server binary: + +**`ServerBody`** — implements `BodyStream` using `http-body-util`. Wraps `Full`, `Empty`, or a streaming `reqwest::Response` for backend responses. Backend response bodies remain as reqwest's streaming type until consumed, avoiding unnecessary buffering. + +**`ReqwestBackendClient`** — implements `BackendClient` using `reqwest`. Sends signed requests to backing object stores with connection pooling (`pool_max_idle_per_host = 20`). + +**`server::run()`** — starts a Hyper HTTP server that accepts connections and delegates to `ProxyHandler` with a `DefaultResolver`. Supports both path-style (`/bucket/key`) and virtual-hosted-style (`bucket.s3.example.com/key`) routing via the resolver's `virtual_host_domain` setting. + +## Module Overview + +``` +src/ +├── lib.rs Crate root +├── body.rs ServerBody implementing BodyStream +├── client.rs ReqwestBackendClient implementing BackendClient +├── server.rs Hyper server setup, request routing +└── bin/ + └── s3-proxy.rs CLI binary entry point +``` + +## Binary Usage + +```bash +cargo build --release -p s3-proxy-server + +# Minimal +./target/release/s3-proxy --config config.toml + +# Full options +./target/release/s3-proxy \ + --config /etc/s3-proxy/config.toml \ + --listen 0.0.0.0:9000 \ + --domain s3.local + +# Environment variable for log level +RUST_LOG=s3_proxy=debug ./target/release/s3-proxy --config config.toml +``` + +## Docker + +```bash +docker build -t s3-proxy . +docker run -v ./config.toml:/etc/s3-proxy/config.toml -p 8080:8080 s3-proxy +``` + +## Using a Different Config Provider + +The default binary uses `StaticProvider` (TOML file) wrapped in `CachedProvider`. The `run()` function accepts any `ConfigProvider` and wraps it in a `DefaultResolver` internally. To use a different provider, modify the binary or write your own: + +```rust +use s3_proxy_core::config::cached::CachedProvider; +use s3_proxy_core::config::http::HttpProvider; // requires config-http feature +use s3_proxy_server::server::{run, ServerConfig}; +use std::time::Duration; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let base = HttpProvider::new( + "https://config-api.internal:8080".into(), + Some("Bearer my-token".into()), + ); + let config = CachedProvider::new(base, Duration::from_secs(300)); + + run(config, ServerConfig::default()).await +} +``` + +## Using a Custom Request Resolver + +For full control over request routing and authorization, you can bypass `run()` and wire up a `ProxyHandler` with a custom `RequestResolver` directly. This is useful when your URL namespace doesn't follow the standard S3 bucket/key pattern, or when authorization is handled by an external service. + +```rust +use s3_proxy_core::proxy::ProxyHandler; +use s3_proxy_core::resolver::{RequestResolver, ResolvedAction}; +use s3_proxy_core::error::ProxyError; + +#[derive(Clone)] +struct MyResolver { /* ... */ } + +impl RequestResolver for MyResolver { + async fn resolve( + &self, + method: &http::Method, + path: &str, + query: Option<&str>, + headers: &http::HeaderMap, + ) -> Result { + // Custom routing: parse the URL, authenticate, authorize, + // and return a ResolvedAction::Proxy or ResolvedAction::Response. + todo!() + } +} + +// Then create the handler directly: +let client = ReqwestBackendClient::new(); +let handler = ProxyHandler::new(client, MyResolver::new()); + +// Use handler.handle_request() in your Hyper service. +``` + +See `s3-proxy-cf-workers/src/source_resolver.rs` for a complete example. + +## Streaming Behavior + +For **GET/HEAD** responses, the backend response body stays as a `reqwest::Response` (streaming) and is forwarded to the client. The proxy does not buffer the full object in memory. + +For **PUT** request bodies, the current implementation collects the incoming body before forwarding. A follow-up optimization would pipe the incoming Hyper body directly to the reqwest request body using `reqwest::Body::wrap_stream`. + +For **multipart uploads**, each part is individually streamed. The `CompleteMultipartUpload` request body (a small XML manifest) is the only body the proxy fully reads and parses. diff --git a/crates/runtimes/server/src/bin/s3-proxy.rs b/crates/runtimes/server/src/bin/s3-proxy.rs new file mode 100644 index 0000000..9f0dcd2 --- /dev/null +++ b/crates/runtimes/server/src/bin/s3-proxy.rs @@ -0,0 +1,54 @@ +//! S3 Proxy Server binary. +//! +//! Usage: +//! s3-proxy --config config.toml [--listen 0.0.0.0:8080] [--domain s3.local] + +use s3_proxy_core::config::cached::CachedProvider; +use s3_proxy_core::config::static_file::StaticProvider; +use s3_proxy_server::server::{run, ServerConfig}; +use std::net::SocketAddr; +use std::time::Duration; + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "s3_proxy=info".into()), + ) + .init(); + + let args: Vec = std::env::args().collect(); + + let config_path = args + .iter() + .position(|a| a == "--config") + .and_then(|i| args.get(i + 1)) + .map(|s| s.as_str()) + .unwrap_or("config.toml"); + + let listen_addr: SocketAddr = args + .iter() + .position(|a| a == "--listen") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or_else(|| ([0, 0, 0, 0], 8080).into()); + + let domain = args + .iter() + .position(|a| a == "--domain") + .and_then(|i| args.get(i + 1)) + .cloned(); + + tracing::info!(config = %config_path, listen = %listen_addr, "starting s3-proxy"); + + let base_config = StaticProvider::from_file(config_path)?; + let config = CachedProvider::new(base_config, Duration::from_secs(60)); + + let server_config = ServerConfig { + listen_addr, + virtual_host_domain: domain, + }; + + run(config, server_config).await +} diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs new file mode 100644 index 0000000..0f6bb0b --- /dev/null +++ b/crates/runtimes/server/src/body.rs @@ -0,0 +1,63 @@ +//! Server-side body type implementing `BodyStream`. + +use bytes::Bytes; +use http_body_util::{BodyExt, Full, Empty}; +use s3_proxy_core::stream::BodyStream; + +/// A body type for the server runtime. +/// +/// Wraps either an incoming request body or a constructed response body. +/// Uses `http_body_util` types which integrate natively with Hyper. +pub enum ServerBody { + /// A body constructed from known bytes. + Full(Full), + /// An empty body. + Empty(Empty), + /// A streaming body from reqwest (for backend responses). + Streaming(reqwest::Response), +} + +/// Error type for server body operations. +#[derive(Debug, thiserror::Error)] +pub enum ServerBodyError { + #[error("hyper error: {0}")] + Hyper(String), + #[error("reqwest error: {0}")] + Reqwest(#[from] reqwest::Error), +} + +impl BodyStream for ServerBody { + type Error = ServerBodyError; + + async fn read_to_bytes(self) -> Result { + match self { + ServerBody::Full(full) => { + let collected = full.collect().await.map_err(|e| ServerBodyError::Hyper(e.to_string()))?; + Ok(collected.to_bytes()) + } + ServerBody::Empty(_) => Ok(Bytes::new()), + ServerBody::Streaming(resp) => { + resp.bytes().await.map_err(ServerBodyError::Reqwest) + } + } + } + + fn from_bytes(bytes: Bytes) -> Self { + ServerBody::Full(Full::new(bytes)) + } + + fn empty() -> Self { + ServerBody::Empty(Empty::new()) + } + + fn content_length(&self) -> Option { + match self { + ServerBody::Full(f) => { + use hyper::body::Body; + f.size_hint().exact() + } + ServerBody::Empty(_) => Some(0), + ServerBody::Streaming(resp) => resp.content_length(), + } + } +} diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs new file mode 100644 index 0000000..ba45e3f --- /dev/null +++ b/crates/runtimes/server/src/client.rs @@ -0,0 +1,93 @@ +//! Backend client using reqwest for outbound HTTP requests. + +use crate::body::ServerBody; +use s3_proxy_core::backend::{BackendClient, BackendRequest, BackendResponse}; +use s3_proxy_core::error::ProxyError; + +/// Backend client that uses `reqwest` to make outbound requests. +/// +/// This keeps the response body as a `reqwest::Response` which can be +/// streamed back to the client without buffering. +#[derive(Clone)] +pub struct ReqwestBackendClient { + client: reqwest::Client, +} + +impl ReqwestBackendClient { + pub fn new() -> Self { + Self { + client: reqwest::Client::builder() + .pool_max_idle_per_host(20) + .build() + .expect("failed to build reqwest client"), + } + } +} + +impl Default for ReqwestBackendClient { + fn default() -> Self { + Self::new() + } +} + +impl BackendClient for ReqwestBackendClient { + type Body = ServerBody; + + async fn send_request( + &self, + request: BackendRequest, + ) -> Result, ProxyError> { + tracing::debug!( + method = %request.method, + url = %request.url, + "server: sending backend request via reqwest" + ); + + let mut req_builder = self.client.request(request.method, &request.url); + + // Set headers + for (key, value) in request.headers.iter() { + req_builder = req_builder.header(key, value); + } + + // Set body + req_builder = match request.body { + ServerBody::Full(full) => { + use http_body_util::BodyExt; + let bytes = full + .collect() + .await + .map_err(|e| ProxyError::BackendError(e.to_string()))? + .to_bytes(); + req_builder.body(bytes) + } + ServerBody::Empty(_) => req_builder, + ServerBody::Streaming(resp) => { + let bytes = resp + .bytes() + .await + .map_err(|e| ProxyError::BackendError(e.to_string()))?; + req_builder.body(bytes) + } + }; + + let response = req_builder + .send() + .await + .map_err(|e| { + tracing::error!(error = %e, "reqwest backend request failed"); + ProxyError::BackendError(e.to_string()) + })?; + + let status = response.status().as_u16(); + let headers = response.headers().clone(); + + tracing::debug!(status = status, "server: backend response received"); + + Ok(BackendResponse { + status, + headers, + body: ServerBody::Streaming(response), + }) + } +} diff --git a/crates/runtimes/server/src/lib.rs b/crates/runtimes/server/src/lib.rs new file mode 100644 index 0000000..4672988 --- /dev/null +++ b/crates/runtimes/server/src/lib.rs @@ -0,0 +1,12 @@ +//! Tokio/Hyper runtime for the S3 proxy gateway. +//! +//! This crate provides concrete implementations of the core traits for a +//! standard server environment using Tokio and Hyper. +//! +//! - [`body::ServerBody`] — implements `BodyStream` using `http-body-util` +//! - [`client::HyperBackendClient`] — implements `BackendClient` using `reqwest` +//! - [`server::run`] — starts the Hyper HTTP server + +pub mod body; +pub mod client; +pub mod server; diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs new file mode 100644 index 0000000..6f06ec9 --- /dev/null +++ b/crates/runtimes/server/src/server.rs @@ -0,0 +1,141 @@ +//! HTTP server using Hyper, wiring everything together. + +use crate::body::ServerBody; +use crate::client::ReqwestBackendClient; +use bytes::Bytes; +use s3_proxy_core::stream::BodyStream; +use http::{Request, Response}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use s3_proxy_core::config::ConfigProvider; +use s3_proxy_core::proxy::ProxyHandler; +use s3_proxy_core::resolver::DefaultResolver; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::net::TcpListener; + +/// Server configuration. +pub struct ServerConfig { + pub listen_addr: SocketAddr, + /// The base domain for virtual-hosted-style requests (e.g., "s3.example.com"). + /// If set, requests to `{bucket}.s3.example.com` use virtual-hosted style. + pub virtual_host_domain: Option, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + listen_addr: ([0, 0, 0, 0], 8080).into(), + virtual_host_domain: None, + } + } +} + +/// Run the S3 proxy server. +/// +/// # Example +/// +/// ```rust,ignore +/// use s3_proxy_core::config::static_file::StaticProvider; +/// use s3_proxy_server::server::{run, ServerConfig}; +/// +/// #[tokio::main] +/// async fn main() { +/// let config = StaticProvider::from_file("config.toml").unwrap(); +/// let server_config = ServerConfig { +/// listen_addr: ([0, 0, 0, 0], 8080).into(), +/// virtual_host_domain: Some("s3.local".to_string()), +/// }; +/// run(config, server_config).await.unwrap(); +/// } +/// ``` +pub async fn run

(config: P, server_config: ServerConfig) -> Result<(), Box> +where + P: ConfigProvider + Send + Sync + 'static, +{ + let client = ReqwestBackendClient::new(); + let resolver = DefaultResolver::new(config, server_config.virtual_host_domain); + let handler = Arc::new(ProxyHandler::new(client, resolver)); + + let listener = TcpListener::bind(server_config.listen_addr).await?; + tracing::info!("listening on {}", server_config.listen_addr); + + loop { + let (stream, remote_addr) = listener.accept().await?; + let handler = handler.clone(); + + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let handler = handler.clone(); + + async move { + tracing::debug!( + remote_addr = %remote_addr, + method = %req.method(), + uri = %req.uri(), + "incoming connection" + ); + let result = handle_hyper_request(req, &handler).await; + match result { + Ok(resp) => Ok::<_, hyper::Error>(resp), + Err(e) => { + tracing::error!(remote_addr = %remote_addr, error = %e, "handler error"); + let body = Full::new(Bytes::from(format!("Internal error: {}", e))); + Ok(Response::builder() + .status(500) + .body(body) + .unwrap()) + } + } + } + }); + + if let Err(err) = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await + { + tracing::error!(remote_addr = %remote_addr, error = %err, "connection error"); + } + }); + } +} + +async fn handle_hyper_request( + req: Request, + handler: &ProxyHandler, +) -> Result>, Box> +where + R: s3_proxy_core::resolver::RequestResolver + Send + Sync, +{ + let method = req.method().clone(); + let uri = req.uri().clone(); + let path = uri.path(); + let query = uri.query(); + let headers = req.headers().clone(); + + // Convert incoming body to ServerBody + let incoming_bytes = req.into_body().collect().await?.to_bytes(); + let body = ServerBody::from_bytes(incoming_bytes); + + let result = handler + .handle_request(method, path, query, &headers, body) + .await; + + // Convert ProxyResult to hyper Response + let mut response = Response::builder().status(result.status); + + for (key, value) in result.headers.iter() { + response = response.header(key, value); + } + + // Get the response body bytes + let body_bytes = match result.body { + ServerBody::Streaming(resp) => resp.bytes().await.unwrap_or_default(), + ServerBody::Full(full) => full.collect().await.map(|c| c.to_bytes()).unwrap_or_default(), + ServerBody::Empty(_) => Bytes::new(), + }; + + Ok(response.body(Full::new(body_bytes))?) +} diff --git a/deploy/.gitignore b/deploy/.gitignore deleted file mode 100644 index f60797b..0000000 --- a/deploy/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.js -!jest.config.js -*.d.ts -node_modules - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/deploy/.npmignore b/deploy/.npmignore deleted file mode 100644 index c1d6d45..0000000 --- a/deploy/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -*.ts -!*.d.ts - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/deploy/.nvmrc b/deploy/.nvmrc deleted file mode 100644 index 2bd5a0a..0000000 --- a/deploy/.nvmrc +++ /dev/null @@ -1 +0,0 @@ -22 diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index b88e456..0000000 --- a/deploy/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Deployment - -This directory contains deployment tooling to create and manage the AWS infrastructure for the Source Data Proxy. - -It embraces an "Infrastructure as Code" approach via [AWS CDK](https://docs.aws.amazon.com/cdk/). Deployments should be triggered via Github Actions. diff --git a/deploy/bin/deploy.ts b/deploy/bin/deploy.ts deleted file mode 100644 index 5287996..0000000 --- a/deploy/bin/deploy.ts +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env node -import * as cdk from "aws-cdk-lib"; -import { DataProxyStack } from "../lib/data-proxy-stack"; -import { Tags } from "aws-cdk-lib"; - -const stage = process.env.STAGE || "dev"; -const vpcId = process.env.VPC_ID; -if (!vpcId) { - throw new Error("VPC_ID is not set"); -} -const certificateArn = process.env.CERTIFICATE_ARN; -if (!certificateArn) { - throw new Error("CERTIFICATE_ARN is not set"); -} -const taskCount = process.env.TASK_COUNT || 1; -const sourceApiUrl = process.env.SOURCE_API_URL || "https://s2.source.coop"; - -const app = new cdk.App(); -const stack = new DataProxyStack(app, `DataProxy-${stage}`, { - vpcId, - proxyDomain: `vercel-api-${stage}.internal`, - proxyDesiredCount: Number(taskCount), - sourceApiUrl, - env: { - account: process.env.AWS_ACCOUNT_ID, - region: process.env.AWS_REGION, - }, - certificateArn, -}); - -Tags.of(stack).add("Cfn-Stack", stack.stackName, { - applyToLaunchedInstances: true, -}); diff --git a/deploy/cdk.json b/deploy/cdk.json deleted file mode 100644 index 8dbbaa8..0000000 --- a/deploy/cdk.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "app": "npx ts-node --prefer-ts-exts bin/deploy.ts", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "**/*.d.ts", - "**/*.js", - "tsconfig.json", - "package*.json", - "yarn.lock", - "node_modules", - "test" - ] - }, - "context": { - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, - "@aws-cdk/core:enablePartitionLiterals": true, - "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, - "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, - "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, - "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, - "@aws-cdk/aws-route53-patters:useCertificate": true, - "@aws-cdk/customresources:installLatestAwsSdkDefault": false, - "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, - "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, - "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, - "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, - "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, - "@aws-cdk/aws-redshift:columnId": true, - "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, - "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, - "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, - "@aws-cdk/aws-kms:aliasNameRef": true, - "@aws-cdk/aws-kms:applyImportedAliasPermissionsToPrincipal": true, - "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, - "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, - "@aws-cdk/aws-efs:denyAnonymousAccess": true, - "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, - "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, - "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, - "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, - "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, - "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, - "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, - "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, - "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, - "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, - "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, - "@aws-cdk/aws-eks:nodegroupNameAttribute": true, - "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, - "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, - "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, - "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, - "@aws-cdk/core:explicitStackTags": true, - "@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature": false, - "@aws-cdk/aws-ecs:disableEcsImdsBlocking": true, - "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, - "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, - "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, - "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, - "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, - "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, - "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, - "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, - "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, - "@aws-cdk/aws-route53-targets:userPoolDomainNameMethodWithoutCustomResource": true, - "@aws-cdk/aws-elasticloadbalancingV2:albDualstackWithoutPublicIpv4SecurityGroupRulesDefault": true, - "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, - "@aws-cdk/core:enableAdditionalMetadataCollection": true, - "@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy": false, - "@aws-cdk/aws-s3:setUniqueReplicationRoleName": true, - "@aws-cdk/aws-events:requireEventBusPolicySid": true, - "@aws-cdk/core:aspectPrioritiesMutating": true, - "@aws-cdk/aws-dynamodb:retainTableReplica": true, - "@aws-cdk/aws-stepfunctions:useDistributedMapResultWriterV2": true, - "@aws-cdk/s3-notifications:addS3TrustKeyPolicyForSnsSubscriptions": true, - "@aws-cdk/aws-ec2:requirePrivateSubnetsForEgressOnlyInternetGateway": true, - "@aws-cdk/aws-s3:publicAccessBlockedByDefault": true, - "@aws-cdk/aws-lambda:useCdkManagedLogGroup": true - } -} diff --git a/deploy/lib/data-proxy-stack.ts b/deploy/lib/data-proxy-stack.ts deleted file mode 100644 index 180cd94..0000000 --- a/deploy/lib/data-proxy-stack.ts +++ /dev/null @@ -1,38 +0,0 @@ -import * as cdk from "aws-cdk-lib"; -import { aws_ec2 as ec2 } from "aws-cdk-lib"; -import { Construct } from "constructs"; -import { VercelApiProxy } from "./vercel-api-proxy"; -import { SourceDataProxy } from "./source-data-proxy"; - -interface DataProxyStackProps extends cdk.StackProps { - vpcId: string; - proxyDomain: string; - sourceApiUrl: string; - proxyDesiredCount: number; - certificateArn: string; -} - -export class DataProxyStack extends cdk.Stack { - constructor(scope: Construct, id: string, props: DataProxyStackProps) { - super(scope, id, props); - - const vpc = ec2.Vpc.fromLookup(this, "vpc", { vpcId: props.vpcId }); - - // Create Vercel API proxy (existing functionality) - const vercelApiProxy = new VercelApiProxy(this, "vercel-api-proxy", { - vpc, - proxyDomain: props.proxyDomain, - }); - - new SourceDataProxy(this, "source-data-proxy", { - vpc, - environment: { - RUST_LOG: "info", - SOURCE_API_PROXY_URL: vercelApiProxy.url, - SOURCE_API_URL: props.sourceApiUrl, - }, - desiredCount: props.proxyDesiredCount, - certificateArn: props.certificateArn, - }); - } -} diff --git a/deploy/lib/source-data-proxy.ts b/deploy/lib/source-data-proxy.ts deleted file mode 100644 index b739585..0000000 --- a/deploy/lib/source-data-proxy.ts +++ /dev/null @@ -1,127 +0,0 @@ -import * as cdk from "aws-cdk-lib"; -import { - aws_ec2 as ec2, - aws_ecs as ecs, - aws_ecs_patterns as ecs_patterns, - aws_logs as logs, - aws_secretsmanager as secretsmanager, - aws_elasticloadbalancingv2 as elbv2, -} from "aws-cdk-lib"; -import { Certificate } from "aws-cdk-lib/aws-certificatemanager"; -import { Construct } from "constructs"; - -interface SourceDataProxyProps { - vpc: ec2.IVpc; - desiredCount: number; - environment: Record; - certificateArn: string; -} - -export class SourceDataProxy extends Construct { - public readonly service: ecs_patterns.ApplicationLoadBalancedFargateService; - - constructor(scope: Construct, id: string, props: SourceDataProxyProps) { - super(scope, id); - - const stack = cdk.Stack.of(this); - - const cluster = new ecs.Cluster(this, "cluster", { - clusterName: `${stack.stackName}-cluster`, - vpc: props.vpc, - enableFargateCapacityProviders: true, - containerInsightsV2: ecs.ContainerInsights.ENHANCED, - }); - - const sourceApiKeySecret = new secretsmanager.Secret( - this, - "source-api-key", - { - secretName: `${stack.stackName}-source-api-key`, - description: - "API Key used to make authenticated requests to the Source API on Vercel", - } - ); - - // Create Application Load Balanced Fargate Service using the pattern - this.service = new ecs_patterns.ApplicationLoadBalancedFargateService( - this, - "service", - { - serviceName: `${stack.stackName}-proxy`, - cluster, - cpu: 4 * 1024, // 4 vCPU - desiredCount: props.desiredCount, - memoryLimitMiB: 12 * 1024, // 12 GB - taskImageOptions: { - image: ecs.ContainerImage.fromAsset("../", { - buildArgs: { - BUILDPLATFORM: "linux/amd64", - TARGETPLATFORM: "linux/amd64", - }, - }), - containerPort: 8080, - family: `${stack.stackName}-proxy`, - environment: props.environment, - secrets: { - SOURCE_API_KEY: ecs.Secret.fromSecretsManager(sourceApiKeySecret), - }, - logDriver: ecs.LogDrivers.awsLogs({ - streamPrefix: "ecs", - logGroup: new logs.LogGroup(this, "log-group", { - logGroupName: `/ecs/${stack.stackName}-proxy`, - retention: logs.RetentionDays.ONE_MONTH, - }), - mode: ecs.AwsLogDriverMode.NON_BLOCKING, - maxBufferSize: cdk.Size.mebibytes(25), - }), - }, - runtimePlatform: { - cpuArchitecture: ecs.CpuArchitecture.X86_64, - operatingSystemFamily: ecs.OperatingSystemFamily.LINUX, - }, - publicLoadBalancer: true, - loadBalancerName: `${stack.stackName}-alb`, - protocol: elbv2.ApplicationProtocol.HTTPS, - listenerPort: 443, - certificate: Certificate.fromCertificateArn( - this, - "certificate", - props.certificateArn - ), - enableExecuteCommand: true, - circuitBreaker: { rollback: true }, - assignPublicIp: true, - capacityProviderStrategies: [ - { - capacityProvider: "FARGATE_SPOT", - // Prefer spot instances over on-demand instances - weight: 2, - }, - { - capacityProvider: "FARGATE", - // Use on-demand instances as a fallback - weight: 1, - }, - ], - } - ); - - if (this.service.taskDefinition.executionRole) { - sourceApiKeySecret.grantRead(this.service.taskDefinition.executionRole); - } - - // Output the ALB DNS name - new cdk.CfnOutput(this, "alb-dns", { - value: this.service.loadBalancer.loadBalancerDnsName, - description: "Application Load Balancer DNS name", - exportName: `${cdk.Stack.of(this).stackName}-alb-dns`, - }); - - // Output the service name - new cdk.CfnOutput(this, "service-name", { - value: this.service.service.serviceName, - description: "ECS Service name", - exportName: `${cdk.Stack.of(this).stackName}-service-name`, - }); - } -} diff --git a/deploy/lib/vercel-api-proxy.ts b/deploy/lib/vercel-api-proxy.ts deleted file mode 100644 index d8e22e6..0000000 --- a/deploy/lib/vercel-api-proxy.ts +++ /dev/null @@ -1,113 +0,0 @@ -import * as cdk from "aws-cdk-lib"; -import { - aws_ec2 as ec2, - aws_iam as iam, - aws_route53 as route53, - aws_route53_targets as route53_targets, -} from "aws-cdk-lib"; -import { Construct } from "constructs"; - -interface VercelApiProxyProps { - vpc: ec2.IVpc; - proxyDomain: string; -} - -export class VercelApiProxy extends Construct { - public readonly url: string; - /** - * To work around Vercel's firewall, we must proxy all requests for the Proxy API through - * a Squid proxy. This will allow us to have a stable IP address for the Proxy API which - * we can add to the Vercel firewall's bypass list. This allows us to retain ephemeral IP - * addresses for the Proxy API and to avoid using other techniques like passing data - * through a NAT Gateway which would have considerable cost implications. - */ - constructor(scope: Construct, id: string, props: VercelApiProxyProps) { - super(scope, id); - - const proxyPort = 3128; - - // Create security group for the proxy - const proxySg = new ec2.SecurityGroup(this, "proxy-sg", { - vpc: props.vpc, - description: "Allow inbound from ECS for Squid proxy", - allowAllOutbound: true, - }); - - // Allow ECS (internal) traffic on port 3128 - proxySg.addIngressRule( - ec2.Peer.ipv4(props.vpc.vpcCidrBlock), - ec2.Port.tcp(proxyPort), - "Allow ECS to connect to Squid" - ); - - // Squid install and minimal config - const userData = ec2.UserData.forLinux(); - userData.addCommands( - "yum update -y", - "yum install -y squid", - - // Write squid.conf using heredoc - "cat <<'EOF' > /etc/squid/squid.conf", - `http_port ${proxyPort}`, - "acl all src 0.0.0.0/0", - "http_access allow all", - "EOF", - - // Enable and start Squid - "systemctl enable squid", - "systemctl restart squid" - ); - - // Enable SSM access for the EC2 instance - const ssmRole = new iam.Role(this, "ec2-ssm-role", { - assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"), - managedPolicies: [ - iam.ManagedPolicy.fromAwsManagedPolicyName( - "AmazonSSMManagedInstanceCore" - ), - ], - }); - - // Launch EC2 instance - const instance = new ec2.Instance(this, "squid-proxy", { - vpc: props.vpc, - role: ssmRole, - instanceType: ec2.InstanceType.of( - ec2.InstanceClass.T3, - ec2.InstanceSize.MICRO - ), - machineImage: ec2.MachineImage.latestAmazonLinux2023(), - vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, - securityGroup: proxySg, - userData, - }); - - // Allocate and associate Elastic IP - const eip = new ec2.CfnEIP(this, "proxy-eip", { - domain: "vpc", - tags: [ - { - key: "Name", - value: `${cdk.Stack.of(this).stackName}-proxy-eip`, - }, - ], - }); - new ec2.CfnEIPAssociation(this, "proxy-eip-assoc", { - allocationId: eip.attrAllocationId, - instanceId: instance.instanceId, - }); - - // Route 53 Private Hosted Zone - const zone = new route53.PrivateHostedZone(this, "proxy-zone", { - vpc: props.vpc, - zoneName: props.proxyDomain, - }); - new route53.ARecord(this, "proxy-a-record", { - zone, - target: route53.RecordTarget.fromIpAddresses(instance.instancePrivateIp), - ttl: cdk.Duration.seconds(60), - }); - - this.url = `http://${props.proxyDomain}:${proxyPort}`; - } -} diff --git a/deploy/package-lock.json b/deploy/package-lock.json deleted file mode 100644 index 788c7e3..0000000 --- a/deploy/package-lock.json +++ /dev/null @@ -1,678 +0,0 @@ -{ - "name": "deploy", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "deploy", - "version": "0.1.0", - "dependencies": { - "aws-cdk-lib": "2.206.0", - "constructs": "^10.0.0" - }, - "bin": { - "deploy": "bin/deploy.js" - }, - "devDependencies": { - "@types/node": "22.7.9", - "aws-cdk": "2.1023.0", - "ts-node": "^10.9.2", - "typescript": "~5.6.3" - } - }, - "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.242", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.242.tgz", - "integrity": "sha512-4c1bAy2ISzcdKXYS1k4HYZsNrgiwbiDzj36ybwFVxEWZXVAP0dimQTCaB9fxu7sWzEjw3d+eaw6Fon+QTfTIpQ==", - "license": "Apache-2.0" - }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", - "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", - "license": "Apache-2.0" - }, - "node_modules/@aws-cdk/cloud-assembly-schema": { - "version": "45.2.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-45.2.0.tgz", - "integrity": "sha512-5TTUkGHQ+nfuUGwKA8/Yraxb+JdNUh4np24qk/VHXmrCMq+M6HfmGWfhcg/QlHA2S5P3YIamfYHdQAB4uSNLAg==", - "bundleDependencies": [ - "jsonschema", - "semver" - ], - "license": "Apache-2.0", - "dependencies": { - "jsonschema": "~1.4.1", - "semver": "^7.7.2" - }, - "engines": { - "node": ">= 18.0.0" - } - }, - "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { - "version": "1.4.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { - "version": "7.7.2", - "inBundle": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", - "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", - "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "22.7.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.9.tgz", - "integrity": "sha512-jrTfRC7FM6nChvU7X2KqcrgquofrWLFDeYC1hKfwNWomVvrn7JIksqf344WN2X/y8xrgqBd2dJATZV4GbatBfg==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.19.2" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.3.4", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", - "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true, - "license": "MIT" - }, - "node_modules/aws-cdk": { - "version": "2.1023.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1023.0.tgz", - "integrity": "sha512-DWMA+IrAsBUNF2RvH7ujpDp7wSJkqTkRL8yfK4AYpEjoGY1KMaKIfxz3M3+Nk3ogM7VhZiW3OGWEOgyDF47HOQ==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "cdk": "bin/cdk" - }, - "engines": { - "node": ">= 18.0.0" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/aws-cdk-lib": { - "version": "2.206.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.206.0.tgz", - "integrity": "sha512-WQGSSzSX+CvIG3j4GICxCAARGaB2dbB2ZiAn8dqqWdUkF6G9pedlSd3bjB0NHOqrxJMu3jYQCYf3gLYTaJuR8A==", - "bundleDependencies": [ - "@balena/dockerignore", - "case", - "fs-extra", - "ignore", - "jsonschema", - "minimatch", - "punycode", - "semver", - "table", - "yaml", - "mime-types" - ], - "license": "Apache-2.0", - "dependencies": { - "@aws-cdk/asset-awscli-v1": "2.2.242", - "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", - "@aws-cdk/cloud-assembly-schema": "^45.0.0", - "@balena/dockerignore": "^1.0.2", - "case": "1.6.3", - "fs-extra": "^11.3.0", - "ignore": "^5.3.2", - "jsonschema": "^1.5.0", - "mime-types": "^2.1.35", - "minimatch": "^3.1.2", - "punycode": "^2.3.1", - "semver": "^7.7.2", - "table": "^6.9.0", - "yaml": "1.10.2" - }, - "engines": { - "node": ">= 14.15.0" - }, - "peerDependencies": { - "constructs": "^10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { - "version": "1.0.2", - "inBundle": true, - "license": "Apache-2.0" - }, - "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.17.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/aws-cdk-lib/node_modules/ansi-regex": { - "version": "5.0.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/ansi-styles": { - "version": "4.3.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/aws-cdk-lib/node_modules/astral-regex": { - "version": "2.0.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/balanced-match": { - "version": "1.0.2", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/brace-expansion": { - "version": "1.1.12", - "inBundle": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/aws-cdk-lib/node_modules/case": { - "version": "1.6.3", - "inBundle": true, - "license": "(MIT OR GPL-3.0-or-later)", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/color-convert": { - "version": "2.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/color-name": { - "version": "1.1.4", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/concat-map": { - "version": "0.0.1", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/emoji-regex": { - "version": "8.0.0", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { - "version": "3.1.3", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/fast-uri": { - "version": "3.0.6", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "inBundle": true, - "license": "BSD-3-Clause" - }, - "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.3.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/aws-cdk-lib/node_modules/graceful-fs": { - "version": "4.2.11", - "inBundle": true, - "license": "ISC" - }, - "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.3.2", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { - "version": "1.0.0", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/jsonfile": { - "version": "6.1.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/aws-cdk-lib/node_modules/jsonschema": { - "version": "1.5.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { - "version": "4.4.2", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/mime-db": { - "version": "1.52.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/aws-cdk-lib/node_modules/mime-types": { - "version": "2.1.35", - "inBundle": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/aws-cdk-lib/node_modules/minimatch": { - "version": "3.1.2", - "inBundle": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/aws-cdk-lib/node_modules/require-from-string": { - "version": "2.0.2", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.7.2", - "inBundle": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/aws-cdk-lib/node_modules/slice-ansi": { - "version": "4.0.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/aws-cdk-lib/node_modules/string-width": { - "version": "4.2.3", - "inBundle": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/strip-ansi": { - "version": "6.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.9.0", - "inBundle": true, - "license": "BSD-3-Clause", - "dependencies": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/yaml": { - "version": "1.10.2", - "inBundle": true, - "license": "ISC", - "engines": { - "node": ">= 6" - } - }, - "node_modules/constructs": { - "version": "10.4.2", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.2.tgz", - "integrity": "sha512-wsNxBlAott2qg8Zv87q3eYZYgheb9lchtBfjHzzLHtXbttwSrHPs1NNQbBrmbb1YZvYg2+Vh0Dor76w4mFxJkA==", - "license": "Apache-2.0" - }, - "node_modules/create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true, - "license": "ISC" - }, - "node_modules/ts-node": { - "version": "10.9.2", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", - "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/typescript": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", - "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true, - "license": "MIT" - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true, - "license": "MIT" - }, - "node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - } - } -} diff --git a/deploy/package.json b/deploy/package.json deleted file mode 100644 index 2b902bc..0000000 --- a/deploy/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "deploy", - "version": "0.1.0", - "bin": { - "deploy": "bin/deploy.js" - }, - "scripts": { - "build": "tsc", - "watch": "tsc -w", - "cdk": "cdk" - }, - "devDependencies": { - "@types/node": "22.7.9", - "aws-cdk": "2.1023.0", - "ts-node": "^10.9.2", - "typescript": "~5.6.3" - }, - "dependencies": { - "aws-cdk-lib": "2.206.0", - "constructs": "^10.0.0" - } -} diff --git a/deploy/tsconfig.json b/deploy/tsconfig.json deleted file mode 100644 index 28bb557..0000000 --- a/deploy/tsconfig.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "lib": [ - "es2022" - ], - "declaration": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "noImplicitThis": true, - "alwaysStrict": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": false, - "inlineSourceMap": true, - "inlineSources": true, - "experimentalDecorators": true, - "strictPropertyInitialization": false, - "typeRoots": [ - "./node_modules/@types" - ] - }, - "exclude": [ - "node_modules", - "cdk.out" - ] -} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..26e0538 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +services: + # ── MinIO (backing object store) ────────────────────────────────────── + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + ports: + - "9000:9000" # S3 API + - "9001:9001" # Web console + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio-data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + # ── Seed MinIO with example buckets and data ────────────────────────── + minio-init: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: /bin/sh + command: + - -c + - | + mc alias set local http://minio:9000 minioadmin minioadmin + mc mb --ignore-existing local/public-data + mc mb --ignore-existing local/private-uploads + mc anonymous set download local/public-data + echo "Hello from s3-proxy!" | mc pipe local/public-data/hello.txt + echo '{"status":"ok"}' | mc pipe local/public-data/health.json + echo "Secret payload" | mc pipe local/private-uploads/docs/secret.txt + +volumes: + minio-data: diff --git a/scripts/build-push.sh b/scripts/build-push.sh deleted file mode 100755 index 92f28a2..0000000 --- a/scripts/build-push.sh +++ /dev/null @@ -1,4 +0,0 @@ -VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version') -docker buildx build --platform linux/arm64 -t 417712557820.dkr.ecr.us-west-2.amazonaws.com/source-data-proxy:v$VERSION --push . -aws ecr get-login-password --region us-west-2 --profile opendata | docker login --username AWS --password-stdin 417712557820.dkr.ecr.us-west-2.amazonaws.com -docker push 417712557820.dkr.ecr.us-west-2.amazonaws.com/source-data-proxy:v$VERSION diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index c8fc6d8..0000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,47 +0,0 @@ -VERSION=$(git tag --points-at HEAD) -SOURCE_API_URL="https://source.coop" - -# Check if the current commit is a release commit -if [ -z "$VERSION" ]; then - echo "No release tag found for this commit. Are you sure you checked out a release commit?" - exit 1; -fi - -# Check if the image for the current version exists in ECR -if [ -z "$(aws ecr describe-images --repository-name source-data-proxy --image-ids=imageTag=$VERSION --profile opendata 2> /dev/null)" ]; then - echo "Could not find image for version $VERSION in ECR. Did you build and push the image?" - exit 1; -fi - -if [ -z "${SOURCE_KEY}" ]; then - echo "The SOURCE_KEY environment variable is not set" - exit 1; -fi - -echo "Deploying $VERSION..." - -jq --arg api_url "$SOURCE_API_URL" --arg image "417712557820.dkr.ecr.us-west-2.amazonaws.com/source-data-proxy:$VERSION" --arg source_key "$SOURCE_KEY" '(.containerDefinitions[0].environment |= [{"name":"SOURCE_KEY", "value": $source_key},{"name":"SOURCE_API_URL", "value": $api_url}]) | (.containerDefinitions[0].image |= $image)' scripts/task_definition.json > scripts/task_definition_deploy.json - -# Register the task definition -if [ -z "$(aws ecs register-task-definition --cli-input-json "file://scripts/task_definition_deploy.json" --profile opendata --no-cli-auto-prompt 2> /dev/null)" ]; then - echo "Failed to create task definition" - echo "Cleaning Up..." - rm scripts/task_definition_deploy.json - exit 1; -fi - -echo "Created Task Definition" - -TASK_DEFINITION_ARN=$(aws ecs list-task-definitions --family-prefix source-data-proxy --status ACTIVE --profile opendata --query "taskDefinitionArns[-1]" --output text) - -echo "Updating Service..." - -if [ -z "$(aws ecs update-service --cluster SourceCooperative-Prod --service source-data-proxy --task-definition $TASK_DEFINITION_ARN --profile opendata 2> /dev/null)" ]; then - echo "Failed to update service" - echo "Cleaning Up..." - rm scripts/task_definition_deploy.json - exit 1; -fi - -echo "Cleaning Up..." -rm scripts/task_definition_deploy.json diff --git a/scripts/run.sh b/scripts/run.sh deleted file mode 100755 index 71b1457..0000000 --- a/scripts/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -export SOURCE_KEY=foobar -export SOURCE_API_URL=http://localhost:3000 -cargo run diff --git a/scripts/tag-release.sh b/scripts/tag-release.sh deleted file mode 100755 index 84e76f2..0000000 --- a/scripts/tag-release.sh +++ /dev/null @@ -1,29 +0,0 @@ -if [[ $(git status -s) ]]; then - echo "ERROR: Please commit all of your changes before tagging the release." - exit 1 -fi - -echo "What type of bump would you like to do?" -echo "1) Patch" -echo "2) Minor" -echo "3) Major" - -read BUMP_TYPE - -if [ $BUMP_TYPE -eq 1 ]; then - cargo bump patch -elif [ $BUMP_TYPE -eq 2 ]; then - cargo bump minor -elif [ $BUMP_TYPE -eq 3 ]; then - cargo bump major -else - echo "ERROR: Invalid bump type" - exit 1 -fi - -VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version') -git add Cargo.toml -git add Cargo.lock -git commit -m "Bump version to v$VERSION" -git tag -a "v$VERSION" -m "v$VERSION" -git push origin --tags diff --git a/scripts/task_definition.json b/scripts/task_definition.json deleted file mode 100644 index 6373faf..0000000 --- a/scripts/task_definition.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "family": "source-data-proxy", - "containerDefinitions": [ - { - "name": "source-data-proxy", - "image": "", - "cpu": 0, - "portMappings": [ - { - "name": "webserver", - "containerPort": 8080, - "hostPort": 8080, - "protocol": "tcp", - "appProtocol": "http" - } - ], - "essential": true, - "environment": [ - { - "name": "SOURCE_KEY", - "value": "{SOURCE_KEY_VALUE_HERE}" - }, - { - "name": "SOURCE_API_URL", - "value": "{SOURCE_API_URL}" - } - ], - "environmentFiles": [], - "mountPoints": [], - "volumesFrom": [], - "ulimits": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/Source-Data-Proxy", - "mode": "non-blocking", - "awslogs-create-group": "true", - "max-buffer-size": "25m", - "awslogs-region": "us-west-2", - "awslogs-stream-prefix": "ecs" - }, - "secretOptions": [] - }, - "systemControls": [] - } - ], - "taskRoleArn": "arn:aws:iam::417712557820:role/SourceCooperative", - "executionRoleArn": "arn:aws:iam::417712557820:role/ecsTaskExecutionRole", - "networkMode": "awsvpc", - "requiresCompatibilities": ["FARGATE"], - "cpu": "4096", - "memory": "12288", - "runtimePlatform": { - "cpuArchitecture": "ARM64", - "operatingSystemFamily": "LINUX" - } -} diff --git a/src/apis/mod.rs b/src/apis/mod.rs deleted file mode 100644 index 0867ca7..0000000 --- a/src/apis/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod source; - -use crate::{backends::common::Repository, utils::auth::UserIdentity, utils::errors::BackendError}; -use async_trait::async_trait; - -pub struct Account { - pub repositories: Vec, -} - -impl Account { - fn default() -> Account { - Account { - repositories: Vec::new(), - } - } -} - -#[async_trait] -pub trait Api { - async fn get_backend_client( - &self, - account_id: &str, - repository_id: &str, - ) -> Result, BackendError>; - - async fn get_account( - &self, - account_id: String, - user_identity: UserIdentity, - ) -> Result; -} diff --git a/src/apis/source/mod.rs b/src/apis/source/mod.rs deleted file mode 100644 index 583eae5..0000000 --- a/src/apis/source/mod.rs +++ /dev/null @@ -1,691 +0,0 @@ -//! Source API client and data structures for the Source Cooperative platform. -//! -//! This module provides types and functionality for interacting with the Source API, -//! including product management, account handling, and storage backend integration. -//! -//! # Overview -//! -//! The Source Cooperative is a platform for sharing and collaborating on data products. -//! This module defines the core data structures that represent products, accounts, -//! and their associated metadata in the system. -//! -//! # Key Types -//! -//! - [`SourceProduct`] - Main product entity with metadata and configuration -//! - [`SourceProductAccount`] - Account information for product owners -//! - [`SourceProductMetadata`] - Product configuration including mirrors and roles -//! - [`SourceApi`] - API client for interacting with the Source platform -//! -//! # Examples -//! -//! ## Creating a Source API client -//! -//! ```rust -//! use source_data_proxy::apis::source::SourceApi; -//! -//! let api = SourceApi::new( -//! "https://api.source.coop".to_string(), -//! "your-api-key".to_string(), -//! None -//! ); -//! ``` -//! -//! ## Parsing product data from JSON -//! -//! ```rust -//! use serde_json; -//! use source_data_proxy::apis::source::SourceProduct; -//! -//! let json = r#"{ -//! "product_id": "example-product", -//! "account_id": "example-account", -//! "title": "Example Product", -//! "description": "An example product", -//! "created_at": "2023-01-01T00:00:00Z", -//! "updated_at": "2023-01-01T00:00:00Z", -//! "visibility": "public", -//! "disabled": false, -//! "data_mode": "open", -//! "featured": 0, -//! "metadata": { ... }, -//! "account": { ... } -//! }"#; -//! -//! let product: SourceProduct = serde_json::from_str(json)?; -//! ``` - -mod types; - -// Re-export all types -pub use types::*; - -use super::{Account, Api}; -use crate::backends::azure::AzureRepository; -use crate::backends::common::Repository; -use crate::backends::s3::S3Repository; -use crate::utils::api::process_json_response; -use crate::utils::auth::UserIdentity; -use crate::utils::errors::BackendError; -use async_trait::async_trait; -use moka::future::Cache; -use rusoto_core::Region; -use std::sync::Arc; -use std::time::Duration; - -/// Client for interacting with the Source Cooperative API. -/// -/// The `SourceApi` provides methods for managing products, accounts, and storage -/// backends. It includes built-in caching for improved performance and supports -/// both direct API calls and proxy-based requests. -/// -/// # Features -/// -/// - **Caching**: Built-in caching for products, data connections, and permissions -/// - **Multiple Storage Backends**: Support for S3, Azure, GCS, MinIO, and Ceph -/// - **Proxy Support**: Optional proxy configuration for network requests -/// - **Authentication**: API key-based authentication with user identity support -/// -/// # Examples -/// -/// ```rust -/// use source_data_proxy::apis::source::SourceApi; -/// -/// let api = SourceApi::new( -/// "https://api.source.coop".to_string(), -/// "your-api-key".to_string(), -/// None // No proxy -/// ); -/// -/// // Get a product -/// let product = api.get_repository_record("account-id", "product-id").await?; -/// ``` -#[derive(Clone)] -pub struct SourceApi { - /// Base URL for the Source API endpoint - pub endpoint: String, - - /// API key for authenticating requests - api_key: String, - - /// Cache for product data to reduce API calls - product_cache: Arc>, - - /// Cache for data connection configurations - data_connection_cache: Arc>, - - /// Cache for API key credentials - access_key_cache: Arc>, - - /// Cache for user permissions - permissions_cache: Arc>>, - - // API Client - client: reqwest::Client, -} - -#[async_trait] -impl Api for SourceApi { - /// Creates and returns a backend client for a specific repository. - /// - /// This method determines the appropriate storage backend (S3 or Azure) based on - /// the repository's configuration and returns a boxed `Repository` trait object. - /// - /// # Arguments - /// - /// * `account_id` - The ID of the account owning the repository. - /// * `repository_id` - The ID of the repository. - /// - /// # Returns - /// - /// Returns a `Result` containing either a boxed `Repository` trait object - /// or an empty error `()` if the client creation fails. - async fn get_backend_client( - &self, - account_id: &str, - repository_id: &str, - ) -> Result, BackendError> { - let product = self - .get_repository_record(account_id, repository_id) - .await?; - - let Some(repository_data) = product - .metadata - .mirrors - .get(product.metadata.primary_mirror.as_str()) - else { - return Err(BackendError::SourceRepositoryMissingPrimaryMirror); - }; - - let data_connection_id = repository_data.connection_id.clone(); - let data_connection = self.get_data_connection(&data_connection_id).await?; - - match data_connection.details.provider.as_str() { - "s3" => { - let region = - if data_connection.authentication.clone().unwrap().auth_type == "s3_local" { - Region::Custom { - name: data_connection - .details - .region - .clone() - .unwrap_or("us-west-2".to_string()), - endpoint: "http://localhost:5050".to_string(), - } - } else { - Region::Custom { - name: data_connection - .details - .region - .clone() - .unwrap_or("us-east-1".to_string()), - endpoint: format!( - "https://s3.{}.amazonaws.com", - data_connection - .details - .region - .clone() - .unwrap_or("us-east-1".to_string()) - ), - } - }; - - let bucket: String = data_connection.details.bucket.clone().unwrap_or_default(); - let base_prefix: String = data_connection - .details - .base_prefix - .clone() - .unwrap_or_default(); - - let mut prefix = format!("{}{}", base_prefix, repository_data.prefix); - if prefix.ends_with('/') { - prefix = prefix[..prefix.len() - 1].to_string(); - }; - - let auth = data_connection.authentication.clone().unwrap(); - - Ok(Box::new(S3Repository { - account_id: account_id.to_string(), - repository_id: repository_id.to_string(), - region, - bucket, - base_prefix: prefix, - auth_method: auth.auth_type, - access_key_id: auth.access_key_id, - secret_access_key: auth.secret_access_key, - })) - } - "az" => { - let account_name: String = data_connection - .details - .account_name - .clone() - .unwrap_or_default(); - - let container_name: String = data_connection - .details - .container_name - .clone() - .unwrap_or_default(); - - let base_prefix: String = data_connection - .details - .base_prefix - .clone() - .unwrap_or_default(); - - Ok(Box::new(AzureRepository { - account_id: account_id.to_string(), - repository_id: repository_id.to_string(), - account_name, - container_name, - base_prefix: format!("{}{}", base_prefix, repository_data.prefix), - })) - } - err => Err(BackendError::UnexpectedDataConnectionProvider { - provider: err.to_string(), - }), - } - } - - async fn get_account( - &self, - account_id: String, - user_identity: UserIdentity, - ) -> Result { - // Create headers - let mut headers = self.build_source_headers(); - if user_identity.api_key.is_some() { - let api_key = user_identity.api_key.unwrap(); - headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str( - format!("{} {}", api_key.access_key_id, api_key.secret_access_key).as_str(), - ) - .unwrap(), - ); - } - - let response = self - .client - .get(format!("{}/api/v1/products/{}", self.endpoint, account_id)) - .headers(headers) - .send() - .await?; - - let product_list = - process_json_response::(response, BackendError::RepositoryNotFound) - .await?; - let mut account = Account::default(); - - for product in product_list.products { - account.repositories.push(product.product_id); - } - - Ok(account) - } -} - -impl SourceApi { - /// Creates a new Source API client with the specified configuration. - /// - /// # Arguments - /// - /// * `endpoint` - Base URL for the Source API (e.g., "https://api.source.coop") - /// * `api_key` - API key for authenticating requests - /// * `proxy_url` - Optional proxy URL for requests (e.g., "http://proxy:8080") - /// - /// # Examples - /// - /// ```rust - /// use source_data_proxy::apis::source::SourceApi; - /// - /// let api = SourceApi::new( - /// "https://api.source.coop".to_string(), - /// "your-api-key".to_string(), - /// None - /// ); - /// ``` - pub fn new(endpoint: String, api_key: String, proxy_url: Option) -> Self { - let product_cache = Arc::new( - Cache::builder() - .time_to_live(Duration::from_secs(60)) // Set TTL to 60 seconds - .build(), - ); - - let data_connection_cache = Arc::new( - Cache::builder() - .time_to_live(Duration::from_secs(60)) // Set TTL to 60 seconds - .build(), - ); - - let access_key_cache = Arc::new( - Cache::builder() - .time_to_live(Duration::from_secs(60)) // Set TTL to 60 seconds - .build(), - ); - - let permissions_cache = Arc::new( - Cache::builder() - .time_to_live(Duration::from_secs(60)) // Set TTL to 60 seconds - .build(), - ); - - let client = { - let mut client = reqwest::Client::builder() - .user_agent(concat!("source-proxy/", env!("CARGO_PKG_VERSION"))); - if let Some(proxy) = proxy_url { - client = client.proxy(reqwest::Proxy::all(proxy).unwrap()); - } - client.build().unwrap() - }; - - SourceApi { - endpoint, - api_key, - product_cache, - data_connection_cache, - access_key_cache, - permissions_cache, - client, - } - } - - /// Builds the headers for the Source API. - /// - /// # Returns - /// - /// Returns a `reqwest::header::HeaderMap` with the appropriate headers. - fn build_source_headers(&self) -> reqwest::header::HeaderMap { - const CORE_REQUEST_HEADERS: &[(&str, &str)] = &[("accept", "application/json")]; - CORE_REQUEST_HEADERS - .iter() - .map(|(name, value)| { - ( - reqwest::header::HeaderName::from_lowercase(name.as_bytes()).unwrap(), - reqwest::header::HeaderValue::from_str(value).unwrap(), - ) - }) - .collect() - } - - /// Retrieves a product record by account and product ID. - /// - /// This method fetches product information from the Source API, including - /// metadata, account details, and configuration. Results are cached for - /// improved performance. - /// - /// # Arguments - /// - /// * `account_id` - The ID of the account that owns the product - /// * `repository_id` - The ID of the product to retrieve - /// - /// # Returns - /// - /// Returns a `Result` containing either a `SourceProduct` struct with the - /// product information or a `BackendError` if the request fails. - /// - /// # Examples - /// - /// ```rust - /// use source_data_proxy::apis::source::SourceApi; - /// - /// let api = SourceApi::new( - /// "https://api.source.coop".to_string(), - /// "your-api-key".to_string(), - /// None - /// ); - /// - /// let product = api.get_repository_record("example-account", "example-product").await?; - /// println!("Product: {}", product.title); - /// ``` - pub async fn get_repository_record( - &self, - account_id: &str, - repository_id: &str, - ) -> Result { - // Try to get the cached value - let cache_key = format!("{account_id}/{repository_id}"); - - if let Some(cached_repo) = self.product_cache.get(&cache_key).await { - return Ok(cached_repo); - } - - // If not in cache, fetch it - let url = format!( - "{}/api/v1/products/{}/{}", - self.endpoint, account_id, repository_id - ); - let headers = self.build_source_headers(); - let response = self.client.get(url).headers(headers).send().await?; - let repository = - process_json_response::(response, BackendError::RepositoryNotFound) - .await?; - - // Cache the successful result - self.product_cache - .insert(cache_key, repository.clone()) - .await; - Ok(repository) - } - - async fn fetch_data_connection( - &self, - data_connection_id: &str, - ) -> Result { - let mut headers = self.build_source_headers(); - headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str(&self.api_key).unwrap(), - ); - - let response = self - .client - .get(format!( - "{}/api/v1/data-connections/{}", - self.endpoint, data_connection_id - )) - .headers(headers) - .send() - .await?; - process_json_response::(response, BackendError::DataConnectionNotFound) - .await - } - - async fn get_data_connection( - &self, - data_connection_id: &str, - ) -> Result { - if let Some(cached_repo) = self.data_connection_cache.get(data_connection_id).await { - return Ok(cached_repo); - } - - // If not in cache, fetch it - match self.fetch_data_connection(data_connection_id).await { - Ok(data_connection) => { - // Cache the successful result - self.data_connection_cache - .insert(data_connection_id.to_string(), data_connection.clone()) - .await; - Ok(data_connection) - } - Err(e) => Err(e), - } - } - - pub async fn get_api_key(&self, access_key_id: &str) -> Result { - if let Some(cached_secret) = self.access_key_cache.get(access_key_id).await { - return Ok(cached_secret); - } - - // If not in cache, fetch it - if access_key_id.is_empty() { - let secret = APIKey { - access_key_id: "".to_string(), - secret_access_key: "".to_string(), - }; - self.access_key_cache - .insert(access_key_id.to_string(), secret.clone()) - .await; - Ok(secret) - } else { - let secret = self.fetch_api_key(access_key_id.to_string()).await?; - self.access_key_cache - .insert(access_key_id.to_string(), secret.clone()) - .await; - Ok(secret) - } - } - - async fn fetch_api_key(&self, access_key_id: String) -> Result { - // Create headers - let mut headers = self.build_source_headers(); - headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str(&self.api_key).unwrap(), - ); - let response = self - .client - .get(format!( - "{}/api/v1/api-keys/{access_key_id}/auth", - self.endpoint - )) - .headers(headers) - .send() - .await?; - let key = process_json_response::(response, BackendError::ApiKeyNotFound).await?; - - Ok(APIKey { - access_key_id, - secret_access_key: key.secret_access_key, - }) - } - - pub async fn is_authorized( - &self, - user_identity: UserIdentity, - account_id: &str, - repository_id: &str, - permission: RepositoryPermission, - ) -> Result { - let anon: bool = user_identity.api_key.is_none(); - - // Try to get the cached value - let cache_key = if anon { - format!("{account_id}/{repository_id}") - } else { - let api_key = user_identity.clone().api_key.unwrap(); - format!("{}/{}/{}", account_id, repository_id, api_key.access_key_id) - }; - - if let Some(cache_permissions) = self.permissions_cache.get(&cache_key).await { - return Ok(cache_permissions.contains(&permission)); - } - - // If not in cache, fetch it - let permissions = self - .fetch_permission(user_identity.clone(), account_id, repository_id) - .await?; - - // Cache the successful result - self.permissions_cache - .insert(cache_key, permissions.clone()) - .await; - - Ok(permissions.contains(&permission)) - } - - pub async fn assert_authorized( - &self, - user_identity: UserIdentity, - account_id: &str, - repository_id: &str, - permission: RepositoryPermission, - ) -> Result { - let authorized = self - .is_authorized(user_identity, account_id, repository_id, permission) - .await?; - if !authorized { - return Err(BackendError::UnauthorizedError); - } - Ok(authorized) - } - - async fn fetch_permission( - &self, - user_identity: UserIdentity, - account_id: &str, - repository_id: &str, - ) -> Result, BackendError> { - // Create headers - let mut headers = self.build_source_headers(); - if user_identity.api_key.is_some() { - let api_key = user_identity.api_key.unwrap(); - headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str( - format!("{} {}", api_key.access_key_id, api_key.secret_access_key).as_str(), - ) - .unwrap(), - ); - } - - let response = self - .client - .get(format!( - "{}/api/v1/products/{account_id}/{repository_id}/permissions", - self.endpoint - )) - .headers(headers) - .send() - .await?; - - process_json_response::>( - response, - BackendError::RepositoryPermissionsNotFound, - ) - .await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - - #[test] - fn test_json_parsing() { - let json_str = r#" - { - "updated_at": "2023-01-15T10:30:00.000Z", - "metadata": { - "primary_mirror": "aws-us-east-1", - "mirrors": { - "aws-us-east-1": { - "storage_type": "s3", - "is_primary": true, - "connection_id": "aws-connection-123", - "config": { "region": "us-east-1", "bucket": "example-bucket" }, - "prefix": "example-account/sample-product/" - } - }, - "tags": ["example", "test"], - "roles": { - "example-account": { - "granted_at": "2023-01-15T10:30:00.000Z", - "account_id": "example-account", - "role": "admin", - "granted_by": "example-account" - } - } - }, - "created_at": "2023-01-01T00:00:00.000Z", - "disabled": false, - "visibility": "public", - "data_mode": "open", - "account_id": "example-account", - "description": "An example product for testing purposes.", - "product_id": "sample-product", - "featured": 0, - "title": "Sample Product", - "account": { - "identity_id": "12345678-1234-1234-1234-123456789abc", - "metadata_public": { - "domains": [ - { - "created_at": "2023-01-10T12:00:00.000Z", - "domain": "example.com", - "status": "unverified" - } - ], - "location": "Example City" - }, - "updated_at": "2023-01-15T10:30:00.000Z", - "flags": ["create_repositories", "create_organizations"], - "created_at": "2023-01-01T00:00:00.000Z", - "emails": [ - { - "verified": false, - "added_at": "2023-01-01T00:00:00.000Z", - "address": "user@example.com", - "is_primary": true - } - ], - "disabled": false, - "metadata_private": {}, - "account_id": "example-account", - "name": "Example User", - "type": "individual" - } - } - "#; - - match serde_json::from_str::(json_str) { - Ok(_product) => { - println!("✅ JSON parsed successfully!"); - } - Err(e) => { - panic!("❌ JSON parsing failed: {}", e); - } - } - } -} diff --git a/src/apis/source/types.rs b/src/apis/source/types.rs deleted file mode 100644 index 3126b4b..0000000 --- a/src/apis/source/types.rs +++ /dev/null @@ -1,437 +0,0 @@ -//! Data structures and types for the Source Cooperative API. -//! -//! This module contains all the data types, enums, and structures used to interact -//! with the Source Cooperative platform, including products, accounts, permissions, -//! and storage configurations. - -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// Repository access permissions for products. -/// -/// Defines the level of access a user or account has to a specific product. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum RepositoryPermission { - /// Read-only access to the product data - #[serde(rename = "read")] - Read, - /// Read and write access to the product data - #[serde(rename = "write")] - Write, -} - -/// Product visibility levels that control who can discover and access the product. -/// -/// This determines how the product appears in listings and search results. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum ProductVisibility { - /// Product is visible to everyone and appears in public listings - #[serde(rename = "public")] - Public, - /// Product is not listed publicly but can be accessed with direct link - #[serde(rename = "unlisted")] - Unlisted, - /// Product access is restricted to specific users or groups - #[serde(rename = "restricted")] - Restricted, -} - -/// Data access modes that define how users can access the product's data. -/// -/// This controls the business model and access patterns for the product. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum ProductDataMode { - /// Data is freely accessible to anyone - #[serde(rename = "open")] - Open, - /// Data requires a subscription to access - #[serde(rename = "subscription")] - Subscription, - /// Data is private and only accessible to authorized users - #[serde(rename = "private")] - Private, -} - -/// Supported storage backend types for product data mirrors. -/// -/// Each product can have multiple mirrors across different storage providers -/// for redundancy and performance optimization. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum StorageType { - /// Amazon S3 compatible storage - #[serde(rename = "s3")] - S3, - /// Microsoft Azure Blob Storage - #[serde(rename = "azure")] - Azure, - /// Google Cloud Storage - #[serde(rename = "gcs")] - Gcs, - /// MinIO object storage - #[serde(rename = "minio")] - Minio, - /// Ceph distributed storage - #[serde(rename = "ceph")] - Ceph, -} - -/// Account types in the Source Cooperative system. -/// -/// Different account types have different capabilities and metadata structures. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum AccountType { - /// Individual user account - #[serde(rename = "individual")] - Individual, - /// Organization or group account - #[serde(rename = "organization")] - Organization, -} - -/// Domain verification status for account domains. -/// -/// Used to track the verification state of custom domains associated with accounts. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum DomainStatus { - /// Domain has not been verified - #[serde(rename = "unverified")] - Unverified, - /// Domain verification is in progress - #[serde(rename = "pending")] - Pending, - /// Domain has been successfully verified - #[serde(rename = "verified")] - Verified, -} - -/// Methods available for domain verification. -/// -/// Different verification methods provide different levels of security and ease of use. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum VerificationMethod { - /// DNS-based verification using TXT records - #[serde(rename = "dns")] - Dns, - /// HTML-based verification using meta tags - #[serde(rename = "html")] - Html, - /// File-based verification using uploaded files - #[serde(rename = "file")] - File, -} - -/// API key credentials for authenticating with the Source API. -/// -/// Contains the access key ID and secret access key used for API authentication. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct APIKey { - /// The access key ID for API authentication - pub access_key_id: String, - /// The secret access key for API authentication - pub secret_access_key: String, -} - -/// Represents a product in the Source Cooperative system. -/// -/// A product is the main entity that contains data and metadata, similar to a repository -/// in traditional version control systems. Products can have multiple storage mirrors -/// for redundancy and performance optimization. -/// -/// # Examples -/// -/// ```rust -/// use serde_json; -/// -/// let json = r#"{ -/// "product_id": "example-product", -/// "account_id": "example-account", -/// "title": "Example Product", -/// "description": "An example product", -/// "created_at": "2023-01-01T00:00:00Z", -/// "updated_at": "2023-01-01T00:00:00Z", -/// "visibility": "public", -/// "disabled": false, -/// "data_mode": "open", -/// "featured": 0, -/// "metadata": { ... }, -/// "account": { ... } -/// }"#; -/// let product: SourceProduct = serde_json::from_str(json)?; -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProduct { - /// Unique identifier for the product (3-40 chars, lowercase, alphanumeric with hyphens) - pub product_id: String, - - /// ID of the account that owns this product - pub account_id: String, - - /// Human-readable title of the product - pub title: String, - - /// Detailed description of the product - pub description: String, - - /// ISO 8601 timestamp when the product was created - pub created_at: String, - - /// ISO 8601 timestamp when the product was last updated - pub updated_at: String, - - /// Visibility level of the product - pub visibility: ProductVisibility, - - /// Whether the product is disabled - pub disabled: bool, - - /// Data access mode for the product - pub data_mode: ProductDataMode, - - /// Featured status (0 = not featured, 1 = featured) - pub featured: i32, - - /// Product metadata including mirrors, tags, and roles - pub metadata: SourceProductMetadata, - - /// Optional account information - pub account: Option, -} - -/// Metadata for a product including mirrors, tags, and roles. -/// -/// Contains all the configuration and organizational information for a product -/// that doesn't fit into the main product fields. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductMetadata { - /// Map of mirror names to mirror configurations - pub mirrors: HashMap, - - /// Name of the primary mirror (key in the mirrors map) - pub primary_mirror: String, - - /// Optional list of tags associated with the product - pub tags: Option>, -} - -/// Configuration for a storage mirror of a product. -/// -/// Each product can have multiple mirrors across different storage providers -/// for redundancy and performance optimization. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductMirror { - /// Type of storage backend used for this mirror - pub storage_type: StorageType, - - /// ID of the data connection configuration - pub connection_id: String, - - /// Storage prefix/path for this mirror - pub prefix: String, - - /// Storage-specific configuration options - pub config: SourceProductMirrorConfig, - - /// Whether this is the primary mirror for the product - pub is_primary: bool, -} - -/// Storage-specific configuration options for a mirror. -/// -/// Different storage backends require different configuration parameters. -/// All fields are optional and only relevant for specific storage types. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductMirrorConfig { - /// AWS region for S3/GCS storage - pub region: Option, - - /// Bucket name for S3/GCS storage - pub bucket: Option, - - /// Container name for Azure Blob Storage - pub container: Option, - - /// Custom endpoint URL for MinIO/Ceph storage - pub endpoint: Option, -} - -/// Account information associated with a product. -/// -/// Contains the account details of the product owner, including profile information, -/// contact details, and organizational metadata. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductAccount { - /// Unique identifier for the account - pub account_id: String, - - /// Type of account (individual or organization) - #[serde(rename = "type")] - pub account_type: AccountType, - - /// Display name of the account - pub name: String, - - /// Identity provider ID (only for individual accounts) - pub identity_id: Option, - - /// Public metadata visible to other users - pub metadata_public: SourceProductAccountMetadataPublic, - - /// Email addresses associated with the account - pub emails: Option>, - - /// ISO 8601 timestamp when the account was created - pub created_at: String, - - /// ISO 8601 timestamp when the account was last updated - pub updated_at: String, - - /// Whether the account is disabled - pub disabled: bool, - - /// Account capability flags - pub flags: Vec, - - /// Private metadata not visible to other users - pub metadata_private: Option>, -} - -/// Domain verification information for an account. -/// -/// Tracks the verification status and process for custom domains associated with accounts. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccountDomain { - /// The domain name being verified - pub domain: String, - - /// Current verification status of the domain - pub status: DomainStatus, - - /// Method used for verification (if applicable) - pub verification_method: Option, - - /// Token used for verification (if applicable) - pub verification_token: Option, - - /// ISO 8601 timestamp when verification was completed - pub verified_at: Option, - - /// ISO 8601 timestamp when domain was added - pub created_at: String, - - /// ISO 8601 timestamp when verification expires (if applicable) - pub expires_at: Option, -} - -/// Email address information for an account. -/// -/// Tracks email addresses associated with an account, including verification status -/// and primary email designation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceAccountEmail { - /// The email address - pub address: String, - - /// Whether the email address has been verified - pub verified: bool, - - /// ISO 8601 timestamp when verification was completed - pub verified_at: Option, - - /// Whether this is the primary email address for the account - pub is_primary: bool, - - /// ISO 8601 timestamp when the email was added - pub added_at: String, -} - -/// Public metadata for an account. -/// -/// Information that is visible to other users and can be displayed in public profiles. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductAccountMetadataPublic { - /// Optional biographical information - pub bio: Option, - - /// Verified domains associated with the account - pub domains: Option>, - - /// Geographic location of the account holder - pub location: Option, - - /// Owner account ID (for organizational accounts) - pub owner_account_id: Option, - - /// List of admin account IDs (for organizational accounts) - pub admin_account_ids: Option>, - - /// List of member account IDs (for organizational accounts) - pub member_account_ids: Option>, -} - -/// Details about a data connection configuration. -/// -/// Contains provider-specific information about how to connect to storage backends. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DataConnectionDetails { - /// Storage provider type (e.g., "s3", "az") - pub provider: String, - /// Cloud region for the storage service - pub region: Option, - /// Base prefix for all data stored through this connection - pub base_prefix: Option, - /// S3 bucket name (for S3-compatible providers) - pub bucket: Option, - /// Azure storage account name (for Azure) - pub account_name: Option, - /// Azure container name (for Azure) - pub container_name: Option, -} - -/// Authentication configuration for a data connection. -/// -/// Defines how to authenticate with the storage backend. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DataConnectionAuthentication { - /// Type of authentication (e.g., "s3_local", "iam_role") - #[serde(rename = "type")] - pub auth_type: String, - /// Access key ID for credential-based authentication - pub access_key_id: Option, - /// Secret access key for credential-based authentication - pub secret_access_key: Option, -} - -/// Configuration for connecting to external data storage. -/// -/// A data connection defines how products can access external storage backends -/// like S3, Azure Blob Storage, or other object storage systems. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DataConnection { - /// Unique identifier for this data connection - pub data_connection_id: String, - /// Human-readable name for the connection - pub name: String, - /// Template for generating storage prefixes - pub prefix_template: String, - /// Whether this connection only allows read operations - pub read_only: bool, - /// List of data modes that can use this connection - pub allowed_data_modes: Vec, - /// Optional flag required on accounts to use this connection - pub required_flag: Option, - /// Provider-specific connection details - pub details: DataConnectionDetails, - /// Authentication configuration for the connection - pub authentication: Option, -} - -/// List of products with pagination support. -/// -/// Used for API responses that return multiple products. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceProductList { - /// List of products in this page - pub products: Vec, - /// Token for fetching the next page of results - pub next: Option, -} diff --git a/src/backends/azure.rs b/src/backends/azure.rs deleted file mode 100644 index 180d5bc..0000000 --- a/src/backends/azure.rs +++ /dev/null @@ -1,305 +0,0 @@ -use actix_web::http::header::RANGE; -use async_trait::async_trait; -use azure_core::request_options::NextMarker; -use azure_storage::StorageCredentials; -use azure_storage_blobs::container::operations::list_blobs::BlobItem; -use azure_storage_blobs::prelude::*; -use bytes::Bytes; -use core::num::NonZeroU32; -use futures::StreamExt; -use futures_core::Stream; -use reqwest; -use std::pin::Pin; -use time::format_description::well_known::{Rfc2822, Rfc3339}; - -use crate::backends::common::{ - CommonPrefix, CompleteMultipartUploadResponse, Content, CreateMultipartUploadResponse, - GetObjectResponse, HeadObjectResponse, ListBucketResult, Repository, -}; -use crate::utils::core::replace_first; -use crate::utils::errors::BackendError; - -use super::common::{MultipartPart, UploadPartResponse}; - -pub struct AzureRepository { - pub account_id: String, - pub repository_id: String, - pub account_name: String, - pub container_name: String, - pub base_prefix: String, -} - -use chrono::format::strftime::StrftimeItems; -use chrono::{DateTime, FixedOffset}; - -fn rfc2822_to_rfc7231(rfc2822_date: &str) -> Result { - // Parse the RFC2822 date string - let datetime = DateTime::parse_from_rfc2822(rfc2822_date)?; - - // Define the format string for RFC7231 - let format = StrftimeItems::new("%a, %d %b %Y %H:%M:%S GMT"); - - // Convert to UTC and format as RFC7231 - Ok(datetime - .with_timezone(&FixedOffset::east_opt(0).unwrap()) - .format_with_items(format.clone()) - .to_string()) -} - -#[async_trait] -impl Repository for AzureRepository { - async fn get_object( - &self, - key: String, - range: Option, - ) -> Result { - let credentials = StorageCredentials::anonymous(); - - let client = BlobServiceClient::new(self.account_name.to_string(), credentials) - .container_client(&self.container_name); - - let blob_client = client.blob_client(format!( - "{}/{}", - self.base_prefix.trim_end_matches('/'), - key - )); - - let blob = blob_client.get_properties().await?; - let content_type = blob.blob.properties.content_type.to_string(); - let etag = blob.blob.properties.etag.to_string(); - let last_modified = rfc2822_to_rfc7231( - blob.blob - .properties - .last_modified - .format(&Rfc2822) - .unwrap_or_else(|_| String::from("Invalid DateTime")) - .as_str(), - ) - .unwrap_or_else(|_| String::from("Invalid DateTime")); - - let client = reqwest::Client::new(); - - // Start building the request - let mut request = client.get(format!( - "https://{}.blob.core.windows.net/{}/{}/{}", - self.account_name, - self.container_name, - self.base_prefix.trim_end_matches('/'), - key - )); - - // If a range is provided, add it to the headers - if let Some(range_value) = range { - request = request.header(RANGE, range_value); - } - - // Send the request and await the response - let response = request.send().await?; - // Check if the status code is successful - if !response.status().is_success() { - return Err(BackendError::UnexpectedApiError(response.text().await?)); - } - - // Get the byte stream from the response - let content_length = response.content_length(); - let stream = response.bytes_stream(); - let boxed_stream: Pin> + Send>> = - Box::pin(stream); - - Ok(GetObjectResponse { - content_length: content_length.unwrap_or(0), - content_type, - etag, - last_modified, - body: boxed_stream, - }) - } - - async fn delete_object(&self, _key: String) -> Result<(), BackendError> { - Err(BackendError::UnsupportedOperation( - "Delete object is not supported on Azure".to_string(), - )) - } - - async fn create_multipart_upload( - &self, - _key: String, - _content_type: Option, - ) -> Result { - Err(BackendError::UnsupportedOperation( - "Create multipart upload is not supported on Azure".to_string(), - )) - } - - async fn abort_multipart_upload( - &self, - _key: String, - _upload_id: String, - ) -> Result<(), BackendError> { - Err(BackendError::UnsupportedOperation( - "Abort multipart upload is not supported on Azure".to_string(), - )) - } - - async fn complete_multipart_upload( - &self, - _key: String, - _upload_id: String, - _parts: Vec, - ) -> Result { - Err(BackendError::UnsupportedOperation( - "Complete multipart upload is not supported on Azure".to_string(), - )) - } - - async fn upload_multipart_part( - &self, - _key: String, - _upload_id: String, - _part_number: String, - _bytes: Bytes, - ) -> Result { - Err(BackendError::UnsupportedOperation( - "Upload multipart part is not supported on Azure".to_string(), - )) - } - - async fn put_object( - &self, - _key: String, - _bytes: Bytes, - _content_type: Option, - ) -> Result<(), BackendError> { - Err(BackendError::UnsupportedOperation( - "Put object is not supported on Azure".to_string(), - )) - } - - async fn head_object(&self, key: String) -> Result { - let credentials = StorageCredentials::anonymous(); - - // Create a client for anonymous access - let client = BlobServiceClient::new(self.account_name.to_string(), credentials) - .container_client(&self.container_name); - - let blob = client - .blob_client(format!( - "{}/{}", - self.base_prefix.trim_end_matches('/'), - key - )) - .get_properties() - .await?; - - Ok(HeadObjectResponse { - content_length: blob.blob.properties.content_length, - content_type: blob.blob.properties.content_type.to_string(), - etag: blob.blob.properties.etag.to_string(), - last_modified: rfc2822_to_rfc7231( - blob.blob - .properties - .last_modified - .format(&Rfc2822) - .unwrap_or_else(|_| String::from("Invalid DateTime")) - .as_str(), - ) - .unwrap_or_else(|_| String::from("Invalid DateTime")), - }) - } - - async fn list_objects_v2( - &self, - prefix: String, - continuation_token: Option, - delimiter: Option, - max_keys: NonZeroU32, - ) -> Result { - let mut result = ListBucketResult { - name: self.account_id.to_string(), - prefix: prefix.clone(), - key_count: 0, - max_keys: 0, - is_truncated: false, - contents: vec![], - common_prefixes: vec![], - next_continuation_token: None, - }; - - let credentials = StorageCredentials::anonymous(); - - // Create a client for anonymous access - let client = BlobServiceClient::new(self.account_name.to_string(), credentials) - .container_client(&self.container_name); - let search_prefix = format!("{}/{}", self.base_prefix.trim_end_matches('/'), prefix); - - let next_marker = continuation_token.map_or(NextMarker::new("".to_string()), Into::into); - - let query_delmiter = delimiter.unwrap_or_default(); - - // List blobs - let mut stream = client - .list_blobs() - .marker(next_marker) - .prefix(search_prefix) - .max_results(max_keys) - .delimiter(query_delmiter) - .into_stream(); - - if let Some(Ok(blob)) = stream.next().await { - if blob.max_results.is_some() { - result.max_keys = blob.max_results.unwrap() as i64; - } - - if blob.next_marker.is_some() { - result.is_truncated = true; - result.next_continuation_token = Some( - blob.next_marker - .unwrap_or(NextMarker::new("".to_string())) - .as_str() - .to_string(), - ); - } - - for blob_item in blob.blobs.items { - match blob_item { - BlobItem::Blob(b) => { - result.contents.push(Content { - key: replace_first( - b.name, - self.base_prefix.clone().trim_end_matches('/').to_string(), - self.repository_id.to_string(), - ), - last_modified: b - .properties - .last_modified - .format(&Rfc3339) - .unwrap_or_else(|_| String::from("Invalid DateTime")), - etag: b.properties.etag.to_string(), - size: b.properties.content_length as i64, - storage_class: b.properties.blob_type.to_string(), - }); - } - BlobItem::BlobPrefix(bp) => { - result.common_prefixes.push(CommonPrefix { - prefix: replace_first( - bp.name, - self.base_prefix.clone().trim_end_matches('/').to_string(), - self.repository_id.to_string(), - ), - }); - } - } - } - } - - Ok(result) - } - async fn copy_object( - &self, - _copy_identifier_path: String, - _key: String, - _range: Option, - ) -> Result<(), BackendError> { - Ok(()) - } -} diff --git a/src/backends/common.rs b/src/backends/common.rs deleted file mode 100644 index 703db50..0000000 --- a/src/backends/common.rs +++ /dev/null @@ -1,170 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use core::num::NonZeroU32; -use futures_core::Stream; -use serde::Deserialize; -use serde::Serialize; -use std::pin::Pin; - -use reqwest::Error as ReqwestError; -type BoxedReqwestStream = Pin> + Send>>; -use crate::utils::errors::BackendError; - -pub struct GetObjectResponse { - pub content_length: u64, - pub content_type: String, - pub last_modified: String, - pub etag: String, - pub body: BoxedReqwestStream, -} - -pub struct HeadObjectResponse { - pub content_length: u64, - pub content_type: String, - pub last_modified: String, - pub etag: String, -} - -#[derive(Debug, Serialize)] -pub struct CompleteMultipartUploadResponse { - #[serde(rename = "Location")] - pub location: String, - #[serde(rename = "Bucket")] - pub bucket: String, - #[serde(rename = "Key")] - pub key: String, - #[serde(rename = "ETag")] - pub etag: String, -} - -#[async_trait] -pub trait Repository { - async fn delete_object(&self, key: String) -> Result<(), BackendError>; - async fn create_multipart_upload( - &self, - key: String, - content_type: Option, - ) -> Result; - async fn abort_multipart_upload( - &self, - key: String, - upload_id: String, - ) -> Result<(), BackendError>; - async fn complete_multipart_upload( - &self, - key: String, - upload_id: String, - parts: Vec, - ) -> Result; - async fn upload_multipart_part( - &self, - key: String, - upload_id: String, - part_number: String, - bytes: Bytes, - ) -> Result; - async fn put_object( - &self, - key: String, - bytes: Bytes, - content_type: Option, - ) -> Result<(), BackendError>; - async fn get_object( - &self, - key: String, - range: Option, - ) -> Result; - async fn head_object(&self, key: String) -> Result; - async fn list_objects_v2( - &self, - prefix: String, - continuation_token: Option, - delimiter: Option, - max_keys: NonZeroU32, - ) -> Result; - async fn copy_object( - &self, - copy_identifier_path: String, - key: String, - range: Option, - ) -> Result<(), BackendError>; -} - -#[derive(Debug, Serialize)] -pub struct ListBucketResult { - #[serde(rename = "Name")] - pub name: String, - #[serde(rename = "Prefix")] - pub prefix: String, - #[serde(rename = "KeyCount")] - pub key_count: i64, - #[serde(rename = "MaxKeys")] - pub max_keys: i64, - #[serde(rename = "IsTruncated")] - pub is_truncated: bool, - #[serde(rename = "Contents")] - pub contents: Vec, - #[serde(rename = "CommonPrefixes")] - pub common_prefixes: Vec, - #[serde(rename = "NextContinuationToken")] - pub next_continuation_token: Option, -} - -#[derive(Debug, Serialize)] -pub struct Content { - #[serde(rename = "Key")] - pub key: String, - #[serde(rename = "LastModified")] - pub last_modified: String, - #[serde(rename = "ETag")] - pub etag: String, - #[serde(rename = "Size")] - pub size: i64, - #[serde(rename = "StorageClass")] - pub storage_class: String, -} - -#[derive(Debug, Serialize)] -pub struct CommonPrefix { - #[serde(rename = "Prefix")] - pub prefix: String, -} - -#[derive(Debug, Serialize)] -pub struct CreateMultipartUploadResponse { - #[serde(rename = "Bucket")] - pub bucket: String, - #[serde(rename = "Key")] - pub key: String, - #[serde(rename = "UploadId")] - pub upload_id: String, -} - -#[derive(Debug, Serialize)] -pub struct UploadPartResponse { - #[serde(rename = "ETag")] - pub etag: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct MultipartPart { - #[serde(rename = "PartNumber")] - pub part_number: i64, - #[serde(rename = "ETag")] - pub etag: String, - #[serde(rename = "ChecksumCRC32")] - pub checksum_crc32: Option, - #[serde(rename = "ChecksumCRC32C")] - pub checksum_crc32c: Option, - #[serde(rename = "ChecksumSHA1")] - pub checksum_sha1: Option, - #[serde(rename = "ChecksumSHA256")] - pub checksum_sha256: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename = "CompleteMultipartUpload")] -pub struct CompleteMultipartUpload { - #[serde(rename = "Part")] - pub parts: Vec, -} diff --git a/src/backends/mod.rs b/src/backends/mod.rs deleted file mode 100644 index 0fb961c..0000000 --- a/src/backends/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod azure; -pub mod common; -pub mod s3; diff --git a/src/backends/s3.rs b/src/backends/s3.rs deleted file mode 100644 index ec758ad..0000000 --- a/src/backends/s3.rs +++ /dev/null @@ -1,382 +0,0 @@ -use super::common::{MultipartPart, UploadPartResponse}; -use crate::backends::common::{ - CommonPrefix, CompleteMultipartUploadResponse, Content, CreateMultipartUploadResponse, - GetObjectResponse, HeadObjectResponse, ListBucketResult, Repository, -}; -use crate::utils::core::replace_first; -use crate::utils::errors::BackendError; -use actix_web::http::header::RANGE; -use async_trait::async_trait; -use bytes::Bytes; -use chrono::Utc; -use core::num::NonZeroU32; -use futures_core::Stream; -use reqwest; -use rusoto_core::Region; -use rusoto_s3::{ - AbortMultipartUploadRequest, CompleteMultipartUploadRequest, CompletedMultipartUpload, - CompletedPart, CreateMultipartUploadRequest, DeleteObjectRequest, HeadObjectRequest, - ListObjectsV2Request, PutObjectRequest, S3Client, UploadPartRequest, S3, -}; -use std::pin::Pin; - -pub struct S3Repository { - pub account_id: String, - pub repository_id: String, - pub region: Region, - pub bucket: String, - pub base_prefix: String, - pub auth_method: String, - pub access_key_id: Option, - pub secret_access_key: Option, -} - -impl S3Repository { - fn create_client(&self) -> Result { - if self.auth_method == "s3_access_key" { - let credentials = rusoto_credential::StaticProvider::new_minimal( - self.access_key_id.clone().unwrap(), - self.secret_access_key.clone().unwrap(), - ); - Ok(S3Client::new_with( - rusoto_core::request::HttpClient::new().unwrap(), - credentials, - self.region.clone(), - )) - } else if self.auth_method == "s3_ecs_task_role" { - let credentials = rusoto_credential::ContainerProvider::new(); - Ok(S3Client::new_with( - rusoto_core::request::HttpClient::new().unwrap(), - credentials, - self.region.clone(), - )) - } else if self.auth_method == "s3_local" { - let credentials = rusoto_credential::ChainProvider::new(); - Ok(S3Client::new_with( - rusoto_core::request::HttpClient::new().unwrap(), - credentials, - self.region.clone(), - )) - } else { - Err(BackendError::UnsupportedAuthMethod(format!( - "Unsupported auth method: {}", - self.auth_method - ))) - } - } -} - -#[async_trait] -impl Repository for S3Repository { - async fn get_object( - &self, - key: String, - range: Option, - ) -> Result { - let head_object_response = self.head_object(key.clone()).await?; - let client = reqwest::Client::new(); - - let url = if self.auth_method == "s3_local" { - format!( - "http://localhost:5050/{}/{}/{}", - self.bucket, self.base_prefix, key - ) - } else { - format!( - "https://s3.{}.amazonaws.com/{}/{}/{}", - self.region.name(), - self.bucket, - self.base_prefix, - key - ) - }; - // Start building the request - let mut request = client.get(url); - - // If a range is provided, add it to the headers - if let Some(range_value) = range { - request = request.header(RANGE, range_value); - } - - // Send the request and await the response - let response = request.send().await?; - // Get the byte stream from the response - let content_length = response.content_length(); - let stream = response.bytes_stream(); - let boxed_stream: Pin> + Send>> = - Box::pin(stream); - - Ok(GetObjectResponse { - content_length: content_length.unwrap_or(0), - content_type: head_object_response.content_type, - etag: head_object_response.etag, - last_modified: head_object_response.last_modified, - body: boxed_stream, - }) - } - - async fn put_object( - &self, - key: String, - bytes: Bytes, - content_type: Option, - ) -> Result<(), BackendError> { - let client = self.create_client()?; - - let request = PutObjectRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - body: Some(bytes.to_vec().into()), - content_type, - ..Default::default() - }; - - client.put_object(request).await?; - Ok(()) - } - - async fn create_multipart_upload( - &self, - key: String, - content_type: Option, - ) -> Result { - let client = self.create_client()?; - - let request = CreateMultipartUploadRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - content_type, - ..Default::default() - }; - - let result = client.create_multipart_upload(request).await?; - Ok(CreateMultipartUploadResponse { - bucket: self.account_id.clone(), - key: key.clone(), - upload_id: result.upload_id.unwrap(), - }) - } - - async fn abort_multipart_upload( - &self, - key: String, - upload_id: String, - ) -> Result<(), BackendError> { - let client = self.create_client()?; - - let request = AbortMultipartUploadRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - upload_id, - ..Default::default() - }; - - client.abort_multipart_upload(request).await?; - Ok(()) - } - - async fn complete_multipart_upload( - &self, - key: String, - upload_id: String, - parts: Vec, - ) -> Result { - let client = self.create_client()?; - - let request = CompleteMultipartUploadRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - upload_id, - multipart_upload: Some(CompletedMultipartUpload { - parts: Some( - parts - .iter() - .map(|part| CompletedPart { - e_tag: Some(part.etag.clone()), - part_number: Some(part.part_number), - }) - .collect(), - ), - }), - ..Default::default() - }; - - let result = client.complete_multipart_upload(request).await?; - Ok(CompleteMultipartUploadResponse { - location: "".to_string(), - bucket: self.account_id.clone(), - key: key.clone(), - etag: result.e_tag.unwrap(), - }) - } - - async fn upload_multipart_part( - &self, - key: String, - upload_id: String, - part_number: String, - bytes: Bytes, - ) -> Result { - let client = self.create_client()?; - - let request = UploadPartRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - upload_id, - part_number: part_number.parse().unwrap(), - body: Some(bytes.to_vec().into()), - ..Default::default() - }; - - let result = client.upload_part(request).await?; - Ok(UploadPartResponse { - etag: result.e_tag.unwrap(), - }) - } - - async fn delete_object(&self, key: String) -> Result<(), BackendError> { - let client = self.create_client()?; - - let request = DeleteObjectRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - ..Default::default() - }; - - client.delete_object(request).await?; - Ok(()) - } - - async fn head_object(&self, key: String) -> Result { - let client = self.create_client()?; - - let request = HeadObjectRequest { - bucket: self.bucket.clone(), - key: format!("{}/{}", self.base_prefix, key), - ..Default::default() - }; - - let result = client.head_object(request).await?; - - Ok(HeadObjectResponse { - content_length: result.content_length.unwrap_or(0) as u64, - content_type: result.content_type.unwrap_or_else(|| "".to_string()), - etag: result.e_tag.unwrap_or_else(|| "".to_string()), - last_modified: result - .last_modified - .unwrap_or_else(|| Utc::now().to_rfc2822()), - }) - } - - async fn list_objects_v2( - &self, - prefix: String, - continuation_token: Option, - delimiter: Option, - max_keys: NonZeroU32, - ) -> Result { - let client = self.create_client()?; - - let mut request = ListObjectsV2Request { - bucket: self.bucket.clone(), - prefix: Some(format!("{}/{}", self.base_prefix, prefix)), - delimiter, - max_keys: Some(max_keys.get() as i64), - ..Default::default() - }; - - if let Some(token) = continuation_token { - request.continuation_token = Some(token); - } - - let output = client.list_objects_v2(request).await?; - let result = ListBucketResult { - name: self.account_id.to_string(), - prefix: format!("{}/{}", self.repository_id, prefix), - key_count: output.key_count.unwrap_or(0), - max_keys: output.max_keys.unwrap_or(0), - is_truncated: output.is_truncated.unwrap_or(false), - next_continuation_token: output.next_continuation_token, - contents: output - .contents - .unwrap_or_default() - .iter() - .map(|item| Content { - key: replace_first( - item.key.clone().unwrap_or_default(), - self.base_prefix.clone(), - self.repository_id.to_string(), - ), - last_modified: item - .last_modified - .clone() - .unwrap_or_else(|| Utc::now().to_rfc2822()), - etag: item.e_tag.clone().unwrap_or_default(), - size: item.size.unwrap_or(0), - storage_class: item.storage_class.clone().unwrap_or_default(), - }) - .collect(), - common_prefixes: output - .common_prefixes - .unwrap_or_default() - .iter() - .map(|item| CommonPrefix { - prefix: replace_first( - item.prefix.clone().unwrap_or_default(), - self.base_prefix.clone(), - self.repository_id.to_string(), - ), - }) - .collect(), - }; - - Ok(result) - } - - async fn copy_object( - &self, - copy_identifier_path: String, - key: String, - range: Option, - ) -> Result<(), BackendError> { - let client = self.create_client()?; - - let request = HeadObjectRequest { - bucket: self.bucket.clone(), - key: copy_identifier_path.to_string(), - ..Default::default() - }; - - let result = client.head_object(request).await?; - let url_client = reqwest::Client::new(); - - let url = if self.auth_method == "s3_local" { - format!( - "http://localhost:5050/{}/{}", - self.bucket, copy_identifier_path - ) - } else { - format!( - "https://s3.{}.amazonaws.com/{}/{}", - self.region.name(), - self.bucket, - copy_identifier_path - ) - }; - - let mut request = url_client.get(url); - - if let Some(range_value) = range { - request = request.header(RANGE, range_value); - } - - let response = request.send().await?; - let content_bytes = response - .bytes() - .await - .unwrap_or_else(|_| bytes::Bytes::from(vec![])); - self.put_object(key.clone(), content_bytes, result.content_type) - .await?; - Ok(()) - } -} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index cc05522..0000000 --- a/src/main.rs +++ /dev/null @@ -1,502 +0,0 @@ -mod apis; -mod backends; -mod utils; -use crate::utils::core::{split_at_first_slash, StreamingResponse}; -use actix_cors::Cors; -use actix_web::body::{BodySize, BoxBody, MessageBody}; -use actix_web::error::ErrorInternalServerError; -use actix_web::{ - delete, get, head, http::header::CONTENT_TYPE, http::header::RANGE, middleware, post, put, web, - App, HttpRequest, HttpResponse, HttpServer, Responder, -}; - -use apis::source::{RepositoryPermission, SourceApi}; -use apis::Api; -use backends::common::{CommonPrefix, CompleteMultipartUpload, ListBucketResult}; -use bytes::Bytes; -use core::num::NonZeroU32; -use env_logger::Env; -use futures_util::StreamExt; -use quick_xml::se::to_string_with_root; -use serde::Deserialize; -use serde_xml_rs::from_str; -use std::env; -use std::fmt::Debug; -use std::pin::Pin; -use std::str::from_utf8; -use std::task::{Context, Poll}; -use utils::auth::{LoadIdentity, UserIdentity}; -use utils::errors::BackendError; -const VERSION: &str = env!("CARGO_PKG_VERSION"); - -struct FakeBody { - size: usize, -} - -impl MessageBody for FakeBody { - type Error = actix_web::Error; - - fn size(&self) -> BodySize { - BodySize::Sized(self.size as u64) - } - - fn poll_next( - self: Pin<&mut Self>, - _: &mut Context<'_>, - ) -> Poll>> { - Poll::Ready(None) - } -} - -#[get("/{account_id}/{repository_id}/{key:.*}")] -async fn get_object( - api_client: web::Data, - req: HttpRequest, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - let mut range_start = 0; - let mut is_range_request = false; - - let range = headers - .get(RANGE) - .and_then(|h| h.to_str().ok()) - .and_then(|r| r.strip_prefix("bytes=")) - .and_then(|bytes_range| bytes_range.split_once('-')) - .and_then(|(start, end)| { - start.parse::().ok().map(|s| { - range_start = s; - if end.is_empty() || end.parse::().is_ok() { - is_range_request = true; - Some(format!("bytes={start}-{end}")) - } else { - None - } - }) - }) - .flatten(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Read, - ) - .await?; - - // Found the repository, now try to get the object - let res = client.get_object(key.clone(), range).await?; - - let mut content_length = String::from("*"); - // Remove this if statement to increase performance since it's making an extra request just to get the total content-length - // This is only needed for range requests and in theory, you can return a * in the Content-Range header to indicate that the content length is unknown - if is_range_request { - content_length = client - .head_object(key.clone()) - .await? - .content_length - .to_string(); - } - - let stream = res - .body - .map(|result| result.map_err(|e| ErrorInternalServerError(e.to_string()))); - - let streaming_response = StreamingResponse::new(stream, res.content_length); - let mut response = if is_range_request { - HttpResponse::PartialContent() - } else { - HttpResponse::Ok() - }; - - let mut response = response - .insert_header(("Content-Type", res.content_type)) - .insert_header(("Last-Modified", res.last_modified)) - .insert_header(("Content-Length", res.content_length.to_string())) - .insert_header(("ETag", res.etag)); - - if is_range_request { - response = response - .insert_header(( - "Content-Range", - format!( - "bytes {}-{}/{}", - range_start, - range_start + res.content_length - 1, - content_length - ), - )) - .insert_header(("Access-Control-Expose-Headers", "Content-Range")); - } - - Ok(response.body(streaming_response)) -} - -#[derive(Debug, Deserialize)] -struct DeleteParams { - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[delete("/{account_id}/{repository_id}/{key:.*}")] -async fn delete_object( - api_client: web::Data, - params: web::Query, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.upload_id.is_none() { - // Found the repository, now try to delete the object - client.delete_object(key.clone()).await?; - Ok(HttpResponse::NoContent().finish()) - } else { - client - .abort_multipart_upload(key.clone(), params.upload_id.clone().unwrap()) - .await?; - Ok(HttpResponse::NoContent().finish()) - } -} - -#[derive(Debug, Deserialize)] -struct PutParams { - #[serde(rename = "partNumber")] - part_number: Option, - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[put("/{account_id}/{repository_id}/{key:.*}")] -async fn put_object( - api_client: web::Data, - req: HttpRequest, - bytes: Bytes, - params: web::Query, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.part_number.is_none() && params.upload_id.is_none() { - // Check if this is a server-side copy operation - if let Some(header_copy_identifier) = req.headers().get("x-amz-copy-source") { - let copy_identifier_path = header_copy_identifier.to_str().unwrap_or(""); - client - .copy_object((©_identifier_path).to_string(), key.clone(), None) - .await?; - Ok(HttpResponse::NoContent().finish()) - } else { - // Found the repository, now try to upload the object - client - .put_object( - key.clone(), - bytes, - headers - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .map(|s| s.to_string()), - ) - .await?; - Ok(HttpResponse::NoContent().finish()) - } - } else if params.part_number.is_some() && params.upload_id.is_some() { - let res = client - .upload_multipart_part( - key.clone(), - params.upload_id.clone().unwrap(), - params.part_number.clone().unwrap(), - bytes, - ) - .await?; - Ok(HttpResponse::Ok() - .insert_header(("ETag", res.etag)) - .finish()) - } else { - Err(BackendError::InvalidRequest( - "Must provide both part number and upload id or neither.".to_string(), - )) - } -} - -#[derive(Debug, Deserialize)] -struct PostParams { - uploads: Option, - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[post("/{account_id}/{repository_id}/{key:.*}")] -async fn post_handler( - api_client: web::Data, - req: HttpRequest, - params: web::Query, - mut payload: web::Payload, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.uploads.is_some() { - let res = client - .create_multipart_upload( - key, - headers - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .map(|s| s.to_string()), - ) - .await?; - let serialized = to_string_with_root("InitiateMultipartUploadResult", &res)?; - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) - } else if params.upload_id.is_some() { - let mut body = String::new(); - while let Some(chunk) = payload.next().await { - match chunk { - Ok(chunk) => match from_utf8(&chunk) { - Ok(s) => body.push_str(s), - Err(_) => { - return Err(BackendError::InvalidRequest("Invalid UTF-8".to_string())) - } - }, - Err(err) => return Err(BackendError::UnexpectedApiError(err.to_string())), - } - } - - let upload = from_str::(&body)?; - let res = client - .complete_multipart_upload(key, params.upload_id.clone().unwrap(), upload.parts) - .await?; - let serialized = to_string_with_root("CompleteMultipartUploadResult", &res)?; - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) - } else { - Err(BackendError::InvalidRequest( - "Must provide either uploads or uploadId".to_string(), - )) - } -} - -#[head("/{account_id}/{repository_id}/{key:.*}")] -async fn head_object( - api_client: web::Data, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Read, - ) - .await?; - - let res = client.head_object(key.clone()).await?; - Ok(HttpResponse::Ok() - .insert_header(("Content-Type", res.content_type)) - .insert_header(("Last-Modified", res.last_modified)) - .insert_header(("ETag", res.etag)) - .body(BoxBody::new(FakeBody { - size: res.content_length as usize, - }))) -} - -#[derive(Deserialize)] -struct ListObjectsV2Query { - #[serde(rename = "prefix")] - prefix: Option, - #[serde(rename = "list-type")] - _list_type: u8, - #[serde(rename = "max-keys")] - max_keys: Option, - #[serde(rename = "delimiter")] - delimiter: Option, - #[serde(rename = "continuation-token")] - continuation_token: Option, -} - -#[get("/{account_id}")] -async fn list_objects( - api_client: web::Data, - info: web::Query, - path: web::Path, - user_identity: web::ReqData, -) -> Result { - let account_id = path.into_inner(); - - if info.prefix.clone().is_some_and(|s| s.is_empty()) || info.prefix.is_none() { - let account = api_client - .get_account(account_id.clone(), (*user_identity).clone()) - .await?; - - let repositories = account.repositories; - let mut common_prefixes = Vec::new(); - for repository_id in repositories.iter() { - common_prefixes.push(CommonPrefix { - prefix: format!("{}/", repository_id.clone()), - }); - } - let list_response = ListBucketResult { - name: account_id.clone(), - prefix: "/".to_string(), - key_count: 0, - max_keys: 0, - is_truncated: false, - contents: vec![], - common_prefixes, - next_continuation_token: None, - }; - - let serialized = to_string_with_root("ListBucketResult", &list_response)?; - return Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)); - } - - let path_prefix = info.prefix.clone().unwrap_or("".to_string()); - - let (repository_id, prefix) = split_at_first_slash(&path_prefix); - - let mut max_keys = NonZeroU32::new(1000).unwrap(); - if let Some(mk) = info.max_keys { - max_keys = mk; - } - - let client = api_client - .get_backend_client(&account_id, repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - repository_id, - RepositoryPermission::Read, - ) - .await?; - - // We're listing within a repository, so we need to query the object store backend - let res = client - .list_objects_v2( - prefix.to_string(), - info.continuation_token.clone(), - info.delimiter.clone(), - max_keys, - ) - .await?; - - let serialized = to_string_with_root("ListBucketResult", &res)?; - - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) -} - -#[get("/")] -async fn index() -> impl Responder { - HttpResponse::Ok().body(format!("Source Cooperative Data Proxy v{VERSION}")) -} - -// Main function to set up and run the HTTP server -#[actix_web::main] -async fn main() -> std::io::Result<()> { - let source_api_url = env::var("SOURCE_API_URL").expect("SOURCE_API_URL must be set"); - let source_api_key = env::var("SOURCE_API_KEY").expect("SOURCE_API_KEY must be set"); - let source_api_proxy_url = env::var("SOURCE_API_PROXY_URL").ok(); // Optional proxy for the Source API - let source_api = web::Data::new(SourceApi::new( - source_api_url, - source_api_key, - source_api_proxy_url, - )); - env_logger::init_from_env(Env::default().default_filter_or("info")); - - HttpServer::new(move || { - App::new() - .app_data(web::PayloadConfig::new(1024 * 1024 * 50)) - .app_data(source_api.clone()) - .app_data(web::Data::new(UserIdentity { api_key: None })) - .wrap( - // Configure CORS - Cors::default() - .allow_any_origin() - .allow_any_method() - .allow_any_header() - .supports_credentials() - .block_on_origin_mismatch(false) - .max_age(3600), - ) - .wrap(middleware::NormalizePath::trim()) - .wrap(middleware::DefaultHeaders::new().add(("X-Version", VERSION))) - .wrap(middleware::Logger::default()) - .wrap(LoadIdentity) - // Register the endpoints - .service(get_object) - .service(delete_object) - .service(post_handler) - .service(put_object) - .service(head_object) - .service(list_objects) - .service(index) - }) - .bind("0.0.0.0:8080")? - .run() - .await -} diff --git a/src/utils/api.rs b/src/utils/api.rs deleted file mode 100644 index 708721b..0000000 --- a/src/utils/api.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::utils::errors::BackendError; -use reqwest::{Response, StatusCode}; -use serde::de::DeserializeOwned; - -/// Process a response, handling both success and error cases -pub async fn process_json_response( - response: Response, - not_found_error: BackendError, -) -> Result { - let status = response.status(); - let url = response.url().to_string(); - let text = response - .text() - .await - .unwrap_or_else(|_| "".to_string()); - - if status.is_success() { - match serde_json::from_str::(&text) { - Ok(val) => Ok(val), - Err(err) => { - log::error!("Failed to parse JSON from {}: {}\nBody: {}", url, err, text); - Err(BackendError::JsonParseError { url }) - } - } - } else if status == StatusCode::NOT_FOUND { - Err(not_found_error) - } else { - let is_server_error = status.is_server_error(); - if is_server_error { - log::error!("Server error from {}: {}\nBody: {}", url, status, text); - Err(BackendError::ApiServerError { - url, - status: status.as_u16(), - message: text, - }) - } else { - log::warn!("Client error from {}: {}\nBody: {}", url, status, text); - Err(BackendError::ApiClientError { - url, - status: status.as_u16(), - message: text, - }) - } - } -} diff --git a/src/utils/auth.rs b/src/utils/auth.rs deleted file mode 100644 index dd6b8c2..0000000 --- a/src/utils/auth.rs +++ /dev/null @@ -1,505 +0,0 @@ -use actix_http::header::HeaderMap; -use actix_web::{ - dev::{self, Service, ServiceRequest, ServiceResponse, Transform}, - web, - web::BytesMut, - Error, HttpMessage, -}; -use futures_util::{future::LocalBoxFuture, stream::StreamExt}; -use hex; -use hmac::{Hmac, Mac}; -use percent_encoding::percent_decode_str; -use sha2::{Digest, Sha256}; -use std::{ - borrow::Cow, - collections::BTreeMap, - future::{ready, Ready}, - rc::Rc, -}; -use url::form_urlencoded; - -use crate::apis::source::{APIKey, SourceApi}; -use crate::utils::errors::BackendError; -use async_trait::async_trait; - -#[async_trait] -pub trait ApiKeyProvider: Send + Sync { - async fn get_api_key(&self, access_key_id: &str) -> Result; -} - -#[async_trait] -impl ApiKeyProvider for SourceApi { - async fn get_api_key(&self, access_key_id: &str) -> Result { - self.get_api_key(access_key_id).await - } -} - -#[derive(Clone)] -pub struct UserIdentity { - pub api_key: Option, -} - -pub struct LoadIdentity; - -impl Transform for LoadIdentity -where - S: Service, Error = Error>, - S::Future: 'static, - B: 'static, -{ - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = LoadIdentityMiddleware; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ready(Ok(LoadIdentityMiddleware { - service: Rc::new(service), - })) - } -} - -pub struct LoadIdentityMiddleware { - service: Rc, -} - -impl Service for LoadIdentityMiddleware -where - S: Service, Error = Error> + 'static, - S::Future: 'static, - B: 'static, -{ - type Response = ServiceResponse; - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; - - dev::forward_ready!(service); - - fn call(&self, mut req: ServiceRequest) -> Self::Future { - let svc = self.service.clone(); - - Box::pin(async move { - let mut body = BytesMut::new(); - let mut stream = req.take_payload(); - while let Some(chunk) = stream.next().await { - body.extend_from_slice(&chunk?); - } - - let identity = match load_identity( - req.app_data::>().unwrap(), - req.method().as_str(), - req.path(), - req.headers(), - req.query_string(), - &body, - ) - .await - { - Ok(api_key) => UserIdentity { - api_key: Some(api_key), - }, - Err(_) => UserIdentity { api_key: None }, - }; - - req.extensions_mut().insert(identity); - - let (_, mut payload) = actix_http::h1::Payload::create(true); - - payload.unread_data(body.into()); - req.set_payload(payload.into()); - - let res = svc.call(req).await?; - - Ok(res) - }) - } -} - -async fn load_identity( - source_api: &web::Data, - method: &str, - path: &str, - headers: &HeaderMap, - query_string: &str, - body: &BytesMut, -) -> Result -where - T: ApiKeyProvider, -{ - let Some(auth) = headers.get("Authorization") else { - return Err("No Authorization header found".to_string()); - }; - - let authorization_header = auth.to_str().unwrap(); - let signature_method = authorization_header.split(" ").next().unwrap(); - - if signature_method != "AWS4-HMAC-SHA256" { - return Err("Invalid Signature Algorithm".to_string()); - } - - let parts = authorization_header - .split(",") - .map(|part| part.trim()) - .collect::>(); - - let credential = parts[0].split("Credential=").nth(1).unwrap_or(""); - let signed_headers = parts[1] - .split("SignedHeaders=") - .nth(1) - .unwrap_or("") - .split(";") - .collect(); - let signature = parts[2].split("Signature=").nth(1).unwrap_or(""); - - let parts = credential.split("/").collect::>(); - let access_key_id = parts[0]; - let date = parts[1]; - let region = parts[2]; - let service = parts[3]; - - let Some(content_hash) = headers.get("x-amz-content-sha256") else { - return Err("No x-amz-content-sha256 header found".to_string()); - }; - - let canonical_request = create_canonical_request( - method, - path, - headers, - signed_headers, - query_string, - body, - content_hash.to_str().unwrap(), - ); - let credential_scope = format!("{date}/{region}/{service}/aws4_request"); - - let Some(datetime) = headers.get("x-amz-date") else { - return Err("No x-amz-date header found".to_string()); - }; - - let api_key = source_api - .get_api_key(access_key_id) - .await - .map_err(|e| e.to_string())?; - - let string_to_sign = create_string_to_sign( - &canonical_request, - datetime.to_str().unwrap(), - &credential_scope, - ); - - let calculated_signature = calculate_signature( - api_key.secret_access_key.as_str(), - date, - region, - service, - &string_to_sign, - ); - - if calculated_signature != signature { - Err("Signature mismatch".to_string()) - } else { - Ok(api_key) - } -} - -fn uri_encode(input: &str, encode_forward_slash: bool) -> Cow<'_, str> { - let mut encoded = String::new(); - let chars = input.chars().peekable(); - - for ch in chars { - if (ch == '/' && !encode_forward_slash) - || (ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' || ch == '~') - { - encoded.push(ch); - } else { - for byte in ch.to_string().as_bytes() { - encoded.push_str(&format!("%{byte:02X}")); - } - } - } - - if encoded == input { - Cow::Borrowed(input) - } else { - Cow::Owned(encoded) - } -} - -fn trim(input: &str) -> String { - input.trim().to_string() -} - -fn lowercase(input: &str) -> String { - input.to_lowercase() -} - -fn hmac_sha256(key: &[u8], message: &[u8]) -> Vec { - // Create HMAC-SHA256 instance - let mut mac = Hmac::::new_from_slice(key).expect("HMAC can take key of any size"); - - // Add message to HMAC - mac.update(message); - - // Calculate HMAC - let result = mac.finalize(); - - // Get the result as bytes - result.into_bytes().to_vec() -} - -fn calculate_signature( - key: &str, - date: &str, - region: &str, - service: &str, - string_to_sign: &str, -) -> String { - let k_date = hmac_sha256(format!("AWS4{key}").as_bytes(), date.as_bytes()); - let k_region = hmac_sha256(&k_date, region.as_bytes()); - let k_service = hmac_sha256(&k_region, service.as_bytes()); - let k_signing = hmac_sha256(&k_service, b"aws4_request"); - - hex::encode(hmac_sha256(&k_signing, string_to_sign.as_bytes())) -} - -fn create_string_to_sign( - canonical_request: &str, - datetime: &str, - credential_scope: &str, -) -> String { - format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - datetime, - credential_scope, - hex::encode(Sha256::digest(canonical_request.as_bytes())) - ) -} - -fn create_canonical_request( - method: &str, - path: &str, - headers: &HeaderMap, - signed_headers: Vec<&str>, - query_string: &str, - body: &BytesMut, - content_hash: &str, -) -> String { - let decoded_path = percent_decode_str(path).decode_utf8().unwrap(); - if content_hash == "UNSIGNED-PAYLOAD" { - return format!( - "{}\n{}\n{}\n{}\n{}\n{}", - method, - uri_encode(decoded_path.as_ref(), false), - get_canonical_query_string(query_string), - get_canonical_headers(headers, &signed_headers), - get_signed_headers(&signed_headers), - content_hash - ); - } - format!( - "{}\n{}\n{}\n{}\n{}\n{}", - method, - uri_encode(decoded_path.as_ref(), false), - get_canonical_query_string(query_string), - get_canonical_headers(headers, &signed_headers), - get_signed_headers(&signed_headers), - hash_payload(body) - ) -} - -fn get_canonical_query_string(query_string: &str) -> String { - if query_string.is_empty() { - return String::new(); - } - - let parsed: Vec<(String, String)> = form_urlencoded::parse(query_string.as_bytes()) - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect(); - - let mut sorted_params: Vec<(String, String)> = parsed; - sorted_params.sort_by(|a, b| a.0.cmp(&b.0)); - - let mut encoded_params: Vec = Vec::new(); - - for (key, value) in sorted_params { - let encoded_key = uri_encode(&key, true); - let encoded_value = uri_encode(&value, true); - - encoded_params.push(format!("{encoded_key}={encoded_value}")); - } - - encoded_params.join("&") -} - -fn get_canonical_headers(headers: &HeaderMap, signed_headers: &Vec<&str>) -> String { - let mut canonical_headers = BTreeMap::new(); - - for (name, value) in headers.iter() { - let canonical_name = lowercase(name.as_str()); - let canonical_value = trim(value.to_str().unwrap()); - - if signed_headers.contains(&canonical_name.as_str()) { - canonical_headers - .entry(canonical_name) - .or_insert_with(Vec::new) - .push(canonical_value); - } - } - - canonical_headers - .iter() - .fold(String::new(), |mut output, (name, values)| { - output.push_str(&format!("{}:{}\n", name, values.join(","))); - output - }) -} - -fn get_signed_headers(signed_headers: &Vec<&str>) -> String { - signed_headers - .iter() - .map(|header| lowercase(header)) - .collect::>() - .join(";") -} - -fn hash_payload(body: &BytesMut) -> String { - hex::encode(Sha256::digest(body)) -} - -#[cfg(test)] -mod tests { - use super::*; - use actix_http::header::{HeaderMap, HeaderName, HeaderValue}; - use async_trait::async_trait; - use common_s3_headers::S3HeadersBuilder; - use std::str::FromStr; - use url::Url; - - #[derive(Clone)] - struct TestSourceApi { - api_key: Option, - } - - impl TestSourceApi { - fn new(api_key: Option) -> Self { - Self { api_key } - } - } - - #[async_trait] - impl ApiKeyProvider for TestSourceApi { - async fn get_api_key(&self, _access_key_id: &str) -> Result { - let Some(key) = &self.api_key else { - return Err(BackendError::ApiKeyNotFound); - }; - Ok(key.clone()) - } - } - - fn create_test_source_api(api_key: Option) -> web::Data { - web::Data::new(TestSourceApi::new(api_key)) - } - - #[tokio::test] - async fn test_load_identity_missing_auth_header() { - let headers = HeaderMap::new(); - let source_api = create_test_source_api(None); - - let result = - load_identity(&source_api, "GET", "/test", &headers, "", &BytesMut::new()).await; - - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "No Authorization header found"); - } - - #[tokio::test] - async fn test_load_identity_invalid_signature_method() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_str("Authorization").unwrap(), - HeaderValue::from_str("INVALID Credential=test-key/20240315/us-east-1/s3, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=test-signature").unwrap(), - ); - - let source_api = create_test_source_api(None); - - let result = - load_identity(&source_api, "GET", "/test", &headers, "", &BytesMut::new()).await; - - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "Invalid Signature Algorithm"); - } - - #[tokio::test] - async fn test_load_identity_missing_content_hash() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_str("Authorization").unwrap(), - HeaderValue::from_str("AWS4-HMAC-SHA256 Credential=test-key/20240315/us-east-1/s3, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=test-signature").unwrap(), - ); - - let source_api = create_test_source_api(None); - - let result = - load_identity(&source_api, "GET", "/test", &headers, "", &BytesMut::new()).await; - - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "No x-amz-content-sha256 header found"); - } - - #[tokio::test] - async fn test_load_identity_missing_date() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_str("Authorization").unwrap(), - HeaderValue::from_str("AWS4-HMAC-SHA256 Credential=test-key/20240315/us-east-1/s3, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=test-signature").unwrap(), - ); - headers.insert( - HeaderName::from_str("x-amz-content-sha256").unwrap(), - HeaderValue::from_str("test-hash").unwrap(), - ); - - let source_api = create_test_source_api(None); - - let result = - load_identity(&source_api, "GET", "/test", &headers, "", &BytesMut::new()).await; - - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "No x-amz-date header found"); - } - - #[tokio::test] - async fn test_load_identity_success() { - let api_key = APIKey { - access_key_id: "test-key".to_string(), - secret_access_key: "test-secret".to_string(), - }; - let source_api = create_test_source_api(Some(api_key.clone())); - - let method = "GET"; - let url = Url::parse("https://test.com/test").unwrap(); - let path = url.path(); - - let headers = HeaderMap::from_iter( - S3HeadersBuilder::new(&url) - .set_access_key(api_key.access_key_id.as_str()) - .set_secret_key(api_key.secret_access_key.as_str()) - .set_region("us-east-1") - .set_method("GET") - .set_service("s3") - .build() - .iter() - .map(|(k, v)| { - ( - HeaderName::from_str(k).unwrap(), - HeaderValue::from_str(v.as_str()).unwrap(), - ) - }), - ); - - let result = load_identity(&source_api, method, path, &headers, "", &BytesMut::new()).await; - - assert!(result.is_ok()); - assert_eq!(result.unwrap().access_key_id, "test-key"); - } -} diff --git a/src/utils/core.rs b/src/utils/core.rs deleted file mode 100644 index 6d2d3f2..0000000 --- a/src/utils/core.rs +++ /dev/null @@ -1,99 +0,0 @@ -use actix_web::{ - body::{BodySize, MessageBody}, - web, Error as ActixError, -}; -use futures::Stream; -use pin_project_lite::pin_project; -use std::pin::Pin; -use std::task::{Context, Poll}; - -pin_project! { - pub struct StreamingResponse { - #[pin] - inner: S, - size: u64, - } -} - -impl StreamingResponse { - pub fn new(inner: S, size: u64) -> Self { - Self { inner, size } - } -} - -impl MessageBody for StreamingResponse -where - S: Stream, - S::Item: Into>, -{ - type Error = ActixError; - - fn size(&self) -> BodySize { - BodySize::Sized(self.size) - } - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - let this = self.project(); - match this.inner.poll_next(cx) { - Poll::Ready(Some(item)) => Poll::Ready(Some(item.into())), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -pub fn replace_first(original: String, from: String, to: String) -> String { - match original.find(&from) { - Some(start_index) => { - let mut result = String::with_capacity(original.len()); - result.push_str(&original[..start_index]); - result.push_str(&to); - result.push_str(&original[start_index + from.len()..]); - result - } - None => original, - } -} - -/// Splits a string at the first forward slash ('/') character. -/// -/// This function takes a string as input and returns a tuple of two strings. -/// The first string in the tuple contains the part of the input before the -/// first slash, and the second string contains the part after the first slash. -/// -/// # Arguments -/// -/// * `input` - A String that may or may not contain a forward slash. -/// -/// # Returns -/// -/// A tuple `(String, String)` where: -/// - The first element is the substring before the first slash. -/// - The second element is the substring after the first slash. -/// -/// If there is no slash in the input string, the function returns the entire -/// input as the first element of the tuple and an empty string as the second element. -/// -/// # Examples -/// -/// ``` -/// let (before, after) = split_at_first_slash("path/to/file".to_string()); -/// assert_eq!(before, "path"); -/// assert_eq!(after, "to/file"); -/// -/// let (before, after) = split_at_first_slash("no_slash".to_string()); -/// assert_eq!(before, "no_slash"); -/// assert_eq!(after, ""); -/// ``` -pub fn split_at_first_slash(input: &str) -> (&str, &str) { - match input.find('/') { - Some(index) => { - let (before, after) = input.split_at(index); - (before, &after[1..]) - } - None => (input, ""), - } -} diff --git a/src/utils/errors.rs b/src/utils/errors.rs deleted file mode 100644 index 3adbc9a..0000000 --- a/src/utils/errors.rs +++ /dev/null @@ -1,654 +0,0 @@ -use actix_web::error; -use actix_web::http::StatusCode; -use actix_web::HttpResponse; -use azure_core::{ - error::{Error as AzureError, ErrorKind as AzureErrorKind}, - StatusCode as AzureStatusCode, -}; -use log::error; -use quick_xml::DeError; -use reqwest::Error as ReqwestError; -use rusoto_core::RusotoError; -use rusoto_s3::{ - AbortMultipartUploadError, CompleteMultipartUploadError, CreateMultipartUploadError, - DeleteObjectError, HeadObjectError, ListObjectsV2Error, PutObjectError, UploadPartError, -}; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum BackendError { - #[error("repository not found")] - RepositoryNotFound, - - #[error("failed to fetch repository permissions")] - RepositoryPermissionsNotFound, - - #[error("source repository missing primary mirror")] - SourceRepositoryMissingPrimaryMirror, - - #[error("object not found: {}", .0.clone().unwrap_or_default())] - ObjectNotFound(Option), - - #[error("api key not found")] - ApiKeyNotFound, - - #[error("data connection not found")] - DataConnectionNotFound, - - #[error("invalid request")] - InvalidRequest(String), - - #[error("reqwest error (url {}, message {})", .0.url().map(|u| u.to_string()).unwrap_or("unknown".to_string()), .0.to_string())] - ReqwestError(#[from] ReqwestError), - - #[error("api threw a server error (url {}, status {}, message {})", .url, .status, .message)] - ApiServerError { - url: String, - status: u16, - message: String, - }, - - #[error("api threw a client error (url {}, status {}, message {})", .url, .status, .message)] - ApiClientError { - url: String, - status: u16, - message: String, - }, - - #[error("failed to parse JSON (url {})", .url)] - JsonParseError { url: String }, - - #[error("unexpected data connection provider (provider {})", .provider)] - UnexpectedDataConnectionProvider { provider: String }, - - #[error("unauthorized")] - UnauthorizedError, - - #[error("unexpected API error: {0}")] - UnexpectedApiError(String), - - #[error("unsupported auth method: {0}")] - UnsupportedAuthMethod(String), - - #[error("unsupported operation: {0}")] - UnsupportedOperation(String), - - #[error("xml parse error: {0}")] - XmlParseError(String), - - #[error("azure error: {0}")] - AzureError(AzureError), - - #[error("s3 error: {0}")] - S3Error(String), -} - -impl error::ResponseError for BackendError { - fn error_response(&self) -> HttpResponse { - let status_code = self.status_code(); - let body = match status_code { - e if e.is_client_error() => self.to_string(), - _ => format!("Internal Server Error: {self}"), - }; - if status_code.is_server_error() { - error!("Error: {}", self); - } - HttpResponse::build(status_code).body(body) - } - - fn status_code(&self) -> StatusCode { - match self { - // Pass through client error status codes - BackendError::ApiClientError { status, .. } => { - StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_REQUEST) - } - - // 400 - BackendError::InvalidRequest(_) - | BackendError::UnsupportedAuthMethod(_) - | BackendError::UnsupportedOperation(_) => StatusCode::BAD_REQUEST, - // 401 - BackendError::UnauthorizedError => StatusCode::UNAUTHORIZED, - // 404 - BackendError::RepositoryNotFound - | BackendError::ObjectNotFound(_) - | BackendError::SourceRepositoryMissingPrimaryMirror - | BackendError::ApiKeyNotFound - | BackendError::DataConnectionNotFound => StatusCode::NOT_FOUND, - - // 502 - BackendError::ReqwestError(_) - | BackendError::ApiServerError { .. } - | BackendError::RepositoryPermissionsNotFound - | BackendError::AzureError(_) - | BackendError::S3Error(_) => StatusCode::BAD_GATEWAY, - // 500 - _ => StatusCode::INTERNAL_SERVER_ERROR, - } - } -} - -// Azure API Errors -impl From for BackendError { - fn from(error: AzureError) -> BackendError { - match error.kind() { - AzureErrorKind::HttpResponse { status, error_code } - if *status == AzureStatusCode::NotFound => - { - BackendError::ObjectNotFound(error_code.clone()) - } - _ => BackendError::AzureError(error), - } - } -} - -// S3 API Errors -fn get_rusoto_error_message( - operation: &str, - error: RusotoError, -) -> String { - match error { - RusotoError::Service(e) => format!("{operation} Service Error: {e}"), - RusotoError::HttpDispatch(e) => format!("{operation} HttpDispatch Error: {e}"), - RusotoError::Credentials(e) => format!("{operation} Credentials Error: {e}"), - RusotoError::Validation(e) => format!("{operation} Validation Error: {e}"), - RusotoError::ParseError(e) => format!("{operation} Parse Error: {e}"), - RusotoError::Unknown(e) => format!("{} Unknown Error: status {}", operation, e.status), - RusotoError::Blocking => format!("{operation} Blocking Error"), - } -} -macro_rules! impl_s3_errors { - ($(($error_type:ty, $operation:expr)),* $(,)?) => { - $( - impl From> for BackendError { - fn from(error: RusotoError<$error_type>) -> BackendError { - BackendError::S3Error(get_rusoto_error_message($operation, error)) - } - } - )* - }; -} -impl_s3_errors!( - (DeleteObjectError, "DeleteObject"), - (PutObjectError, "PutObject"), - (CreateMultipartUploadError, "CreateMultipartUpload"), - (AbortMultipartUploadError, "AbortMultipartUpload"), - (CompleteMultipartUploadError, "CompleteMultipartUpload"), - (UploadPartError, "UploadPart"), -); -impl From> for BackendError { - fn from(error: RusotoError) -> BackendError { - match error { - RusotoError::Service(HeadObjectError::NoSuchKey(e)) => { - BackendError::ObjectNotFound(Some(e)) - } - RusotoError::Unknown(e) if e.status == StatusCode::NOT_FOUND => { - BackendError::ObjectNotFound(Some(e.body_as_str().to_string())) - } - _ => BackendError::S3Error(get_rusoto_error_message("HeadObject", error)), - } - } -} -impl From> for BackendError { - fn from(error: RusotoError) -> BackendError { - match error { - RusotoError::Service(ListObjectsV2Error::NoSuchBucket(_)) => { - BackendError::RepositoryNotFound - } - _ => BackendError::S3Error(get_rusoto_error_message("ListObjectsV2", error)), - } - } -} - -impl From for BackendError { - fn from(error: DeError) -> BackendError { - BackendError::XmlParseError(format!("failed to parse xml: {error}")) - } -} -impl From for BackendError { - fn from(error: serde_xml_rs::Error) -> BackendError { - BackendError::XmlParseError(format!("failed to parse xml: {error}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use actix_web::body::to_bytes; - use actix_web::error::ResponseError; - use actix_web::http::StatusCode; - use bytes::Bytes; - use quick_xml::DeError; - use rusoto_core::RusotoError; - use rusoto_s3::{HeadObjectError, ListObjectsV2Error, PutObjectError}; - use serde_xml_rs::Error as XmlError; - - /// Tests for S3 error handling - mod s3_errors { - use super::*; - - #[tokio::test] - async fn should_convert_head_object_no_such_key_to_404() { - let error = RusotoError::Service(HeadObjectError::NoSuchKey("test-key".to_string())); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::ObjectNotFound(_)), - "expected error to be ObjectNotFound" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - let response = backend_error.error_response(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("object not found: test-key") - ); - } - - #[tokio::test] - async fn should_convert_list_objects_no_such_bucket_to_404() { - let error = - RusotoError::Service(ListObjectsV2Error::NoSuchBucket("test-bucket".to_string())); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::RepositoryNotFound), - "expected error to be converted to RepositoryNotFound" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - let response = backend_error.error_response(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("repository not found") - ); - } - - #[tokio::test] - async fn should_convert_put_object_unknown_error_to_502() { - let error: RusotoError = - RusotoError::Unknown(rusoto_core::request::BufferedHttpResponse { - status: StatusCode::INTERNAL_SERVER_ERROR, - headers: Default::default(), - body: Bytes::new(), - }); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::S3Error(_)), - "expected error to be converted to S3Error" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::BAD_GATEWAY, - "expected status code to be 502" - ); - let response = backend_error.error_response(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("Internal Server Error: s3 error: PutObject Unknown Error: status 500 Internal Server Error") - ); - } - } - - /// Tests for Azure error handling - mod azure_errors { - use super::*; - - #[tokio::test] - async fn should_convert_not_found_to_404() { - let error = AzureError::new( - AzureErrorKind::HttpResponse { - status: AzureStatusCode::NotFound, - error_code: Some("ResourceNotFound".to_string()), - }, - "Resource not found", - ); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::ObjectNotFound(_)), - "expected error to be converted to ObjectNotFound" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - let response = backend_error.error_response(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("object not found: ResourceNotFound") - ); - } - - #[tokio::test] - async fn should_convert_other_errors_to_502() { - let error = AzureError::new( - AzureErrorKind::HttpResponse { - status: AzureStatusCode::InternalServerError, - error_code: Some("InternalError".to_string()), - }, - "Internal error", - ); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::AzureError(_)), - "expected error to be converted to AzureError" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::BAD_GATEWAY, - "expected status code to be 502" - ); - let response = backend_error.error_response(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("Internal Server Error: azure error: Internal error") - ); - } - } - - /// Tests for client-side error handling - mod client_errors { - use super::*; - - #[tokio::test] - async fn should_handle_unauthorized_error() { - let error = BackendError::UnauthorizedError; - assert_eq!( - error.status_code(), - StatusCode::UNAUTHORIZED, - "expected status code to be 401" - ); - assert_eq!( - error.to_string(), - "unauthorized", - "expected error message to be 'unauthorized'" - ); - let response = error.error_response(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("unauthorized") - ); - } - - #[tokio::test] - async fn should_handle_invalid_request_error() { - let error = BackendError::InvalidRequest("bad input".to_string()); - assert_eq!( - error.status_code(), - StatusCode::BAD_REQUEST, - "expected status code to be 400" - ); - assert_eq!( - error.to_string(), - "invalid request", - "expected error message to be 'invalid request'" - ); - let response = error.error_response(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("invalid request") - ); - } - - #[tokio::test] - async fn should_handle_unsupported_auth_method() { - let error = BackendError::UnsupportedAuthMethod("basic".to_string()); - assert_eq!( - error.status_code(), - StatusCode::BAD_REQUEST, - "expected status code to be 400" - ); - assert_eq!( - error.to_string(), - "unsupported auth method: basic", - "expected error message to include auth method" - ); - let response = error.error_response(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("unsupported auth method: basic") - ); - } - - #[tokio::test] - async fn should_handle_unsupported_operation() { - let error = BackendError::UnsupportedOperation("delete".to_string()); - assert_eq!( - error.status_code(), - StatusCode::BAD_REQUEST, - "expected status code to be 400" - ); - assert_eq!( - error.to_string(), - "unsupported operation: delete", - "expected error message to include operation" - ); - let response = error.error_response(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert_eq!( - to_bytes(response.into_body()).await.unwrap(), - Bytes::from("unsupported operation: delete") - ); - } - } - - /// Tests for XML parsing errors - mod xml_errors { - use super::*; - - #[test] - fn should_convert_quick_xml_error() { - let error = DeError::UnexpectedStart(b"unexpected start of stream".to_vec()); - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::XmlParseError(_)), - "expected error to be converted to XmlParseError" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::INTERNAL_SERVER_ERROR, - "expected status code to be 500" - ); - } - - #[test] - fn should_convert_serde_xml_error() { - let error = XmlError::Custom { - field: "invalid XML format".to_string(), - }; - let backend_error = BackendError::from(error); - - assert!( - matches!(backend_error, BackendError::XmlParseError(_)), - "expected error to be converted to XmlParseError" - ); - assert_eq!( - backend_error.status_code(), - StatusCode::INTERNAL_SERVER_ERROR, - "expected status code to be 500" - ); - } - } - - /// Tests for API-related errors - mod api_errors { - use super::*; - - #[test] - fn should_handle_api_server_error() { - let error = BackendError::ApiServerError { - url: "https://api.example.com".to_string(), - status: 500, - message: "Internal Server Error".to_string(), - }; - assert_eq!( - error.status_code(), - StatusCode::BAD_GATEWAY, - "expected status code to be 502" - ); - assert!( - error.to_string().contains("api threw a server error"), - "expected error message to mention server error" - ); - } - - #[test] - fn should_handle_api_client_error_400() { - let error = BackendError::ApiClientError { - url: "https://api.example.com".to_string(), - status: 400, - message: "Bad Request".to_string(), - }; - assert_eq!( - error.status_code(), - StatusCode::BAD_REQUEST, - "expected status code to be 400" - ); - assert!( - error.to_string().contains("api threw a client error"), - "expected error message to mention client error" - ); - } - - #[test] - fn should_handle_api_client_error_404() { - let error = BackendError::ApiClientError { - url: "https://api.example.com".to_string(), - status: 404, - message: "Bad Request".to_string(), - }; - assert_eq!( - error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - assert!( - error.to_string().contains("api threw a client error"), - "expected error message to mention client error" - ); - } - - #[test] - fn should_handle_json_parse_error() { - let error = BackendError::JsonParseError { - url: "https://api.example.com".to_string(), - }; - assert_eq!( - error.status_code(), - StatusCode::INTERNAL_SERVER_ERROR, - "expected status code to be 500" - ); - assert!( - error.to_string().contains("failed to parse JSON"), - "expected error message to mention JSON parsing" - ); - } - } - - /// Tests for repository-related errors - mod repository_errors { - use super::*; - - #[test] - fn should_handle_repository_not_found() { - let error = BackendError::RepositoryNotFound; - assert_eq!( - error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - assert_eq!( - error.to_string(), - "repository not found", - "expected error message to be 'repository not found'" - ); - } - - #[test] - fn should_handle_repository_permissions_not_found() { - let error = BackendError::RepositoryPermissionsNotFound; - assert_eq!( - error.status_code(), - StatusCode::BAD_GATEWAY, - "expected status code to be 502" - ); - assert_eq!( - error.to_string(), - "failed to fetch repository permissions", - "expected error message to mention permissions" - ); - } - - #[test] - fn should_handle_source_repository_missing_primary_mirror() { - let error = BackendError::SourceRepositoryMissingPrimaryMirror; - assert_eq!( - error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - assert_eq!( - error.to_string(), - "source repository missing primary mirror", - "expected error message to mention missing mirror" - ); - } - } - - /// Tests for data connection errors - mod data_connection_errors { - use super::*; - - #[test] - fn should_handle_data_connection_not_found() { - let error = BackendError::DataConnectionNotFound; - assert_eq!( - error.status_code(), - StatusCode::NOT_FOUND, - "expected status code to be 404" - ); - assert_eq!( - error.to_string(), - "data connection not found", - "expected error message to be 'data connection not found'" - ); - } - - #[test] - fn should_handle_unexpected_data_connection_provider() { - let error = BackendError::UnexpectedDataConnectionProvider { - provider: "unknown".to_string(), - }; - assert_eq!( - error.status_code(), - StatusCode::INTERNAL_SERVER_ERROR, - "expected status code to be 500" - ); - assert!( - error - .to_string() - .contains("unexpected data connection provider"), - "expected error message to mention unexpected provider" - ); - } - } -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs deleted file mode 100644 index c80fecf..0000000 --- a/src/utils/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod api; -pub mod auth; -pub mod core; -pub mod errors; From 2c0ba67ff3eb45726dcec6b6ba91fa1d73661f41 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Feb 2026 22:08:35 -0800 Subject: [PATCH 02/82] Rework to use objectstore --- .gitignore | 3 +- CLAUDE.md | 25 +- Cargo.lock | 92 ++- Cargo.toml | 7 + README.md | 36 +- crates/libs/core/Cargo.toml | 2 + crates/libs/core/README.md | 20 +- crates/libs/core/src/backend.rs | 70 +- crates/libs/core/src/error.rs | 26 +- crates/libs/core/src/lib.rs | 6 +- crates/libs/core/src/proxy.rs | 613 ++++++++++++++---- crates/libs/core/src/response_body.rs | 37 ++ crates/libs/core/src/stream.rs | 40 -- crates/runtimes/cf-workers/Cargo.toml | 16 +- crates/runtimes/cf-workers/README.md | 15 +- crates/runtimes/cf-workers/src/body.rs | 173 ++--- crates/runtimes/cf-workers/src/client.rs | 116 ++-- .../cf-workers/src/fetch_connector.rs | 162 +++++ crates/runtimes/cf-workers/src/lib.rs | 94 +-- crates/runtimes/server/Cargo.toml | 2 + crates/runtimes/server/README.md | 22 +- crates/runtimes/server/src/body.rs | 97 ++- crates/runtimes/server/src/client.rs | 113 ++-- crates/runtimes/server/src/lib.rs | 4 +- crates/runtimes/server/src/server.rs | 38 +- 25 files changed, 1206 insertions(+), 623 deletions(-) create mode 100644 crates/libs/core/src/response_body.rs delete mode 100644 crates/libs/core/src/stream.rs create mode 100644 crates/runtimes/cf-workers/src/fetch_connector.rs diff --git a/.gitignore b/.gitignore index 96fa024..5666652 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -/target .DS_Store scripts/task_definition.json +target +.wrangler \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index bdc8f2c..1028c93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # S3 Proxy Gateway -Multi-runtime S3 gateway proxy in Rust. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. +Multi-runtime S3 gateway proxy in Rust. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. Uses the `object_store` crate for high-level operations (GET, HEAD, PUT, LIST) and raw signed HTTP for multipart uploads. ## Workspace Structure @@ -25,9 +25,26 @@ cargo test ## Key Architecture Notes -- **RequestResolver pattern**: `ProxyHandler` is generic over a `RequestResolver` trait. The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.handle_request()`. +- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has two methods: `create_store()` returns an `Arc` for high-level operations, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: + - **Server**: `ServerBackend` uses the default `object_store` HTTP connector (reqwest-based) and reqwest for raw HTTP. + - **CF Workers**: `WorkerBackend` uses a custom `FetchConnector` (implementing `object_store::client::HttpConnector`) that bridges the Workers Fetch API to `object_store`, and `web_sys::fetch` for raw HTTP. +- **Operation dispatch**: The proxy handler dispatches S3 operations to different backends: + - **GET** → `store.get_opts()` with Range/If-Match/If-None-Match header parsing; returns `ProxyResponseBody::Stream`. + - **HEAD** → `store.head()`; returns metadata headers + empty body. + - **PUT** → `store.put()`; request body materialized to `Bytes` by the runtime before calling the handler. + - **LIST** → `store.list_with_delimiter()`; builds S3 ListObjectsV2 XML directly from `ListResult` (no XML rewriting needed). `IsTruncated` is always `false` (object_store fetches all pages internally). + - **Multipart** (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload) → raw signed HTTP via `backend.send_raw()` + `S3RequestSigner`. These use raw HTTP because `object_store`'s `MultipartUpload` API manages state internally and doesn't expose upload IDs for stateless proxying. +- **ProxyResponseBody**: A concrete enum (`Stream`, `Bytes`, `Empty`) replacing the old generic `B: BodyStream` type parameter. Runtimes convert this to their native response type at the edge. `Stream` wraps a `BoxStream<'static, Result>`. +- **RequestResolver pattern**: The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.handle_request()`. - **MaybeSend pattern**: Core traits use `MaybeSend`/`MaybeSync` (defined in `crates/libs/core/src/maybe_send.rs`) instead of `Send`/`Sync`. On native targets these resolve to `Send`/`Sync`; on `wasm32` they are no-op blanket traits. This allows the CF Workers runtime to use `!Send` JS interop types (`JsValue`, `ReadableStream`, etc.). +- **FetchConnector** (CF Workers): `crates/runtimes/cf-workers/src/fetch_connector.rs` implements `object_store::client::HttpConnector` and `HttpService` using the Workers Fetch API. Since `worker::Fetch::send()` is `!Send`, each call is wrapped in `spawn_local` with a oneshot channel to bridge back to the `Send` context that `object_store` expects. Response body streaming uses an mpsc channel: a `spawn_local` task reads from the Workers `ByteStream` and sends chunks through the channel, whose receiver is wrapped as an `HttpResponseBody`. +- **Streaming**: The server runtime now streams GET responses (previously buffered). The CF Workers runtime bridges `object_store`'s `BoxStream` to a JS `ReadableStream` via `TransformStream` — a `spawn_local` task reads Rust stream chunks and writes them to the writable side; the readable side is returned in the Response. Bytes cross the WASM boundary in chunks (lazy, not buffered). - **cf-workers is excluded from `default-members`** in the root `Cargo.toml` because WASM types are `!Send` and will fail to compile on native targets. Always use `--target wasm32-unknown-unknown` when working with this crate. -- **Streaming passthrough**: The CF Workers runtime passes `ReadableStream` bodies through opaquely — bytes never enter Rust memory for GET/PUT requests. The `WorkerBody` enum wraps `Bytes`, `ReadableStream`, or `Empty`. - **Config loading** (CF Workers): `PROXY_CONFIG` can be either a JSON string (via `wrangler secret`) or a JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`). Both formats are handled. -- **List response rewriting**: When a resolver returns `ResolvedAction::Proxy` with a `ListRewrite`, the proxy handler buffers the (small) list XML response and rewrites `` and `` element values — stripping a backend prefix and optionally prepending a new one. This is handled in `crates/libs/core/src/s3/list_rewrite.rs`. +- **List response construction**: LIST responses are built directly from `object_store::ListResult` as S3 XML. When a resolver returns a `ListRewrite`, prefix stripping/adding is applied to `ObjectMeta.location` and `common_prefixes` paths before XML generation. The `list_rewrite` module in `crates/libs/core/src/s3/list_rewrite.rs` is retained for backward compatibility. + +## Known Limitations + +1. **Multipart uses raw HTTP**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP, which is S3-specific. +2. **LIST returns all results**: `object_store::list_with_delimiter()` fetches all pages internally. No S3-style pagination (continuation tokens, max-keys truncation). `IsTruncated` is always `false`. +3. **S3 only**: `BucketConfig` and store creation assume S3. Adding GCS/Azure requires a `backend_type` field and builder dispatch. diff --git a/Cargo.lock b/Cargo.lock index 0bad143..1187832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -698,6 +698,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -771,6 +786,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1002,6 +1018,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hyper" version = "0.14.32" @@ -1268,6 +1290,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1472,6 +1503,41 @@ dependencies = [ "libm", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http 1.4.0", + "http-body-util", + "humantime", + "hyper 1.8.1", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.38.4", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1646,6 +1712,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1826,6 +1902,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -1838,6 +1915,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.36", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -2011,13 +2089,19 @@ dependencies = [ name = "s3-proxy-cf-workers" version = "0.1.0" dependencies = [ + "async-trait", "bytes", "chrono", "console_error_panic_hook", + "futures", "getrandom 0.2.17", + "getrandom 0.3.4", "http 1.4.0", + "http-body 1.0.1", + "http-body-util", "js-sys", - "quick-xml", + "object_store", + "quick-xml 0.37.5", "s3-proxy-auth", "s3-proxy-core", "serde", @@ -2040,10 +2124,12 @@ dependencies = [ "base64", "bytes", "chrono", + "futures", "hex", "hmac", "http 1.4.0", - "quick-xml", + "object_store", + "quick-xml 0.37.5", "reqwest", "serde", "serde_json", @@ -2062,10 +2148,12 @@ name = "s3-proxy-server" version = "0.1.0" dependencies = [ "bytes", + "futures", "http 1.4.0", "http-body-util", "hyper 1.8.1", "hyper-util", + "object_store", "reqwest", "s3-proxy-auth", "s3-proxy-core", diff --git a/Cargo.toml b/Cargo.toml index a9ee0c5..5308b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,10 @@ hmac = "0.12" sha2 = { version = "0.10", features = ["oid"] } rsa = "0.9" +# Object store +object_store = { version = "0.12", default-features = false, features = ["aws"] } +futures = "0.3" + # XML quick-xml = { version = "0.37", features = ["serialize"] } @@ -50,6 +54,9 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" aws-sdk-dynamodb = "1" sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres"] } +# HTTP body +http-body = "1" + # Server runtime tokio = { version = "1", features = ["full"] } hyper = { version = "1", features = ["full"] } diff --git a/README.md b/README.md index cdb4a7a..bdd3c9f 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,20 @@ A multi-runtime S3 gateway that streams requests to and from backing object stor ## Architecture -``` -┌──────────────┐ ┌──────────────────────────────────────┐ -│ S3 Clients │────────▶│ s3-proxy-rs │ -│ (aws cli, │ │ │ -│ boto3, │ │ ┌────────────┐ ┌────────────────┐ │ ┌──────────────┐ -│ sdk, etc.) │ │ │ Auth │ │ Config │ │────────▶│ Backend S3 │ -│ │◀────────│ │ (SigV4, │ │ (Static, │ │ │ (AWS, MinIO │ -│ │ │ │ STS, │ │ HTTP API, │ │◀────────│ R2, etc.) │ -│ │ │ │ OIDC) │ │ DynamoDB, │ │ └──────────────┘ -│ │ │ └────────────┘ │ Postgres) │ │ -└──────────────┘ │ └────────────────┘ │ - └──────────────────────────────────────┘ +```mermaid +flowchart LR + Clients["S3 Clients
(aws cli, boto3, sdk, etc.)"] + + subgraph Proxy["s3-proxy-rs"] + Auth["Auth
(SigV4, STS, OIDC)"] + Config["Config
(Static, HTTP API, DynamoDB, Postgres)"] + end + + Backend["Backend S3
(AWS, MinIO, R2, etc.)"] + + Proxy <--> Clients + Backend <--> Proxy + ``` ### Crate Layout @@ -30,7 +32,7 @@ crates/ └── cf-workers/ (s3-proxy-cf-workers) # Cloudflare Workers for edge deployments ``` -Libraries define trait abstractions (`BodyStream`, `BackendClient`, `ConfigProvider`, `RequestResolver`). Runtimes implement those traits with platform-native primitives: Hyper/reqwest on the server, JS Fetch API with `ReadableStream` passthrough on Workers. +Libraries define trait abstractions (`ProxyBackend`, `ConfigProvider`, `RequestResolver`). Runtimes implement `ProxyBackend` with platform-native primitives: the server runtime uses `object_store` with its default HTTP connector and `reqwest` for raw multipart requests; the Workers runtime uses a custom `FetchConnector` that bridges `object_store` to the JS Fetch API. The `RequestResolver` trait decouples "what to do with a request" from the proxy handler. A `DefaultResolver` handles standard S3 proxy behavior (parse, auth, authorize via `ConfigProvider`), while custom resolvers like `SourceCoopResolver` can implement entirely different namespace mapping and authorization schemes. @@ -245,7 +247,7 @@ Wire it into the proxy handler in your runtime: ```rust let resolver = MyResolver::new(/* ... */); -let handler = ProxyHandler::new(backend_client, resolver); +let handler = ProxyHandler::new(backend, resolver); let result = handler.handle_request(method, path, query, &headers, body).await; ``` @@ -314,11 +316,11 @@ The proxy validates the JWT against the OIDC provider's JWKS, checks the trust p The crate workspace separates concerns so the core logic compiles to both native and WASM targets: -**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. +**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. It uses `object_store` for high-level operations (GET, HEAD, PUT, LIST) and a `ProxyBackend` trait for runtime-specific store creation and raw HTTP (multipart). All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. -**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It implements `BodyStream` with `http_body_util` types and `BackendClient` with reqwest's streaming HTTP client. +**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It implements `ProxyBackend` using `object_store`'s default HTTP connector for high-level operations and reqwest for raw multipart requests. GET responses stream from `object_store` through Hyper without buffering. -**s3-proxy-cf-workers** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It implements `BodyStream` wrapping JS `ReadableStream` and `BackendClient` using the Fetch API. The critical optimization: for GET requests, the JS `ReadableStream` from the backend response is passed directly to the outbound worker `Response` — bytes never touch Rust/WASM memory. +**`s3-proxy-cf-workers`** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It implements `ProxyBackend` with a custom `FetchConnector` that bridges `object_store` to the Workers Fetch API via `spawn_local` + channel patterns (since JS interop types are `!Send`). Response body streams are converted to JS `ReadableStream` via `TransformStream` for efficient delivery. ## License diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml index 8144a5c..58799bc 100644 --- a/crates/libs/core/Cargo.toml +++ b/crates/libs/core/Cargo.toml @@ -28,6 +28,8 @@ hmac.workspace = true sha2.workspace = true quick-xml.workspace = true tracing.workspace = true +object_store.workspace = true +futures.workspace = true # Optional config backend deps aws-sdk-dynamodb = { workspace = true, optional = true } diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md index b0c7e1f..b24cd2d 100644 --- a/crates/libs/core/README.md +++ b/crates/libs/core/README.md @@ -8,11 +8,9 @@ The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in conta ## Key Abstractions -The core defines four trait boundaries that runtime crates implement: +The core defines three trait boundaries that runtime crates implement: -**`BodyStream`** — A streaming body type. The core almost never reads body bytes; it passes them through opaquely from client to backend and back. Each runtime provides its own type (Hyper's `Body`, JS `ReadableStream`, etc.). - -**`BackendClient`** — Makes signed outbound HTTP requests to backing object stores. The server runtime uses `reqwest`; the worker runtime uses the JS Fetch API. +**`ProxyBackend`** — Creates `object_store` instances for high-level S3 operations (GET, HEAD, PUT, LIST) and provides a `send_raw()` method for multipart operations that require signed HTTP requests. The server runtime uses `object_store`'s default HTTP connector + reqwest; the worker runtime uses a custom `FetchConnector` that bridges to the JS Fetch API. **`ConfigProvider`** — Retrieves bucket, role, and credential configuration. Ships with four implementations behind feature flags: @@ -34,7 +32,7 @@ Any provider can be wrapped with `CachedProvider` for in-memory TTL caching. ``` src/ ├── auth.rs SigV4 verification, identity resolution, authorization -├── backend.rs BackendClient trait, S3RequestSigner, outbound SigV4 signing +├── backend.rs ProxyBackend trait, S3RequestSigner (multipart), RawResponse ├── config/ │ ├── mod.rs ConfigProvider trait definition │ ├── cached.rs TTL caching wrapper for any provider @@ -49,13 +47,13 @@ src/ │ ├── request.rs Parse incoming HTTP → S3Operation enum │ ├── response.rs Serialize S3 XML responses │ └── list_rewrite.rs Rewrite / values in list response XML -├── stream.rs BodyStream trait +├── response_body.rs ProxyResponseBody enum (Stream, Bytes, Empty) └── types.rs BucketConfig, RoleConfig, StoredCredential, etc. ``` ## Usage -This crate is not used directly. Runtime crates (`s3-proxy-server`, `s3-proxy-cf-workers`) depend on it and provide concrete trait implementations. If you're building a custom runtime integration, depend on this crate and implement `BodyStream`, `BackendClient`, and optionally `ConfigProvider` or `RequestResolver`. +This crate is not used directly. Runtime crates (`s3-proxy-server`, `s3-proxy-cf-workers`) depend on it and provide concrete `ProxyBackend` implementations. If you're building a custom runtime integration, depend on this crate and implement `ProxyBackend`, and optionally `ConfigProvider` or `RequestResolver`. ### Standard usage with a ConfigProvider @@ -66,15 +64,15 @@ use s3_proxy_core::proxy::ProxyHandler; use s3_proxy_core::resolver::DefaultResolver; use s3_proxy_core::config::static_file::StaticProvider; -let backend_client = MyBackendClient::new(); +let backend = MyBackend::new(); let config = StaticProvider::from_file("config.toml")?; let resolver = DefaultResolver::new(config, Some("s3.example.com".into())); -let handler = ProxyHandler::new(backend_client, resolver); +let handler = ProxyHandler::new(backend, resolver); // In your HTTP handler: let result = handler.handle_request(method, path, query, &headers, body).await; -// Convert `result` (ProxyResult) to your runtime's HTTP response. +// Convert `result` (ProxyResult with ProxyResponseBody) to your runtime's HTTP response. ``` ### Custom resolver @@ -103,7 +101,7 @@ impl RequestResolver for MyResolver { } } -let handler = ProxyHandler::new(backend_client, MyResolver::new()); +let handler = ProxyHandler::new(backend, MyResolver::new()); ``` See `s3-proxy-cf-workers/src/source_resolver.rs` for a real-world example that maps a `/{account}/{repo}/{key}` namespace to dynamically-resolved S3 backends with external API authorization. diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index 958d535..966080b 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -1,46 +1,56 @@ -//! Backend client abstraction for making signed requests to backing object stores. +//! Backend abstraction for proxying requests to backing object stores. +//! +//! [`ProxyBackend`] is the main trait runtimes implement. It provides two +//! capabilities: +//! +//! 1. **`create_store()`** — build an `ObjectStore` for high-level operations +//! (GET, HEAD, PUT, LIST) routed through `object_store`. +//! 2. **`send_raw()`** — send a pre-signed HTTP request for operations not +//! covered by `ObjectStore` (multipart uploads). +//! +//! [`S3RequestSigner`] is retained for signing multipart requests. use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; -use crate::stream::BodyStream; +use bytes::Bytes; use http::HeaderMap; +use object_store::ObjectStore; use std::future::Future; +use std::sync::Arc; -/// A fully prepared request to send to a backend object store. -#[derive(Debug)] -pub struct BackendRequest { - pub method: http::Method, - pub url: String, - pub headers: HeaderMap, - pub body: B, -} +use crate::types::BucketConfig; -/// The response from a backend object store. -pub struct BackendResponse { - pub status: u16, - pub headers: HeaderMap, - pub body: B, -} - -/// Trait for making outbound HTTP requests to backing object stores. +/// Trait for runtime-specific backend operations. /// /// Each runtime provides its own implementation: -/// - Server runtime: uses `hyper` client with native async streaming -/// - Worker runtime: uses the Fetch API, keeping JS `ReadableStream` intact -/// -/// The body type `B` is the runtime's native stream type. This ensures -/// zero-copy passthrough: the proxy never materializes the full response -/// body in memory. -pub trait BackendClient: MaybeSend + MaybeSync + 'static { - type Body: BodyStream; - - fn send_request( +/// - Server runtime: uses `reqwest` for raw HTTP, default `object_store` HTTP connector +/// - Worker runtime: uses `web_sys::fetch` for raw HTTP, custom `FetchConnector` for `object_store` +pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { + /// Create an `ObjectStore` for the given bucket configuration. + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; + + /// Send a raw HTTP request (used for multipart operations that + /// `ObjectStore` doesn't expose at the right abstraction level). + fn send_raw( &self, - request: BackendRequest, - ) -> impl Future, ProxyError>> + MaybeSend; + method: http::Method, + url: String, + headers: HeaderMap, + body: Bytes, + ) -> impl Future> + MaybeSend; +} + +/// Response from a raw HTTP request to a backend. +pub struct RawResponse { + pub status: u16, + pub headers: HeaderMap, + pub body: Bytes, } /// Helper to build a signed URL + headers for an outbound request to S3. +/// +/// Used for multipart operations (CreateMultipartUpload, UploadPart, +/// CompleteMultipartUpload, AbortMultipartUpload) that go through raw HTTP. pub struct S3RequestSigner { pub access_key_id: String, pub secret_access_key: String, diff --git a/crates/libs/core/src/error.rs b/crates/libs/core/src/error.rs index 4e19ba7..1d6305a 100644 --- a/crates/libs/core/src/error.rs +++ b/crates/libs/core/src/error.rs @@ -7,6 +7,9 @@ pub enum ProxyError { #[error("bucket not found: {0}")] BucketNotFound(String), + #[error("no such key: {0}")] + NoSuchKey(String), + #[error("access denied")] AccessDenied, @@ -31,6 +34,12 @@ pub enum ProxyError { #[error("backend error: {0}")] BackendError(String), + #[error("precondition failed")] + PreconditionFailed, + + #[error("not modified")] + NotModified, + #[error("config error: {0}")] ConfigError(String), @@ -43,6 +52,7 @@ impl ProxyError { pub fn s3_error_code(&self) -> &'static str { match self { Self::BucketNotFound(_) => "NoSuchBucket", + Self::NoSuchKey(_) => "NoSuchKey", Self::AccessDenied => "AccessDenied", Self::SignatureDoesNotMatch => "SignatureDoesNotMatch", Self::InvalidRequest(_) => "InvalidRequest", @@ -51,6 +61,8 @@ impl ProxyError { Self::InvalidOidcToken(_) => "InvalidIdentityToken", Self::RoleNotFound(_) => "AccessDenied", Self::BackendError(_) => "InternalError", + Self::PreconditionFailed => "PreconditionFailed", + Self::NotModified => "NotModified", Self::ConfigError(_) => "InternalError", Self::Internal(_) => "InternalError", } @@ -59,13 +71,25 @@ impl ProxyError { /// HTTP status code for this error. pub fn status_code(&self) -> u16 { match self { - Self::BucketNotFound(_) => 404, + Self::BucketNotFound(_) | Self::NoSuchKey(_) => 404, Self::AccessDenied | Self::MissingAuth | Self::ExpiredCredentials => 403, Self::SignatureDoesNotMatch => 403, Self::InvalidRequest(_) => 400, Self::InvalidOidcToken(_) => 400, Self::RoleNotFound(_) => 403, + Self::PreconditionFailed => 412, + Self::NotModified => 304, Self::BackendError(_) | Self::ConfigError(_) | Self::Internal(_) => 500, } } + + /// Convert an `object_store::Error` into a `ProxyError`. + pub fn from_object_store_error(e: object_store::Error) -> Self { + match e { + object_store::Error::NotFound { path, .. } => Self::NoSuchKey(path), + object_store::Error::Precondition { .. } => Self::PreconditionFailed, + object_store::Error::NotModified { .. } => Self::NotModified, + _ => Self::BackendError(e.to_string()), + } + } } diff --git a/crates/libs/core/src/lib.rs b/crates/libs/core/src/lib.rs index 39911b6..478ce3f 100644 --- a/crates/libs/core/src/lib.rs +++ b/crates/libs/core/src/lib.rs @@ -8,8 +8,8 @@ //! //! ## Key Abstractions //! -//! - [`stream::BodyStream`] — abstract over response/request body types across runtimes -//! - [`backend::BackendClient`] — make signed outbound requests to backing object stores +//! - [`response_body::ProxyResponseBody`] — concrete response body type (Stream, Bytes, Empty) +//! - [`backend::ProxyBackend`] — create object stores and send raw HTTP requests //! - [`config::ConfigProvider`] — retrieve bucket/role/credential configuration from any backend //! - [`auth`] — SigV4 request verification and credential resolution //! - [`s3::request`] — parse incoming S3 API requests into typed operations @@ -23,6 +23,6 @@ pub mod error; pub mod maybe_send; pub mod proxy; pub mod resolver; +pub mod response_body; pub mod s3; -pub mod stream; pub mod types; diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index c5d2577..2c90af3 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -1,18 +1,18 @@ //! The main proxy handler that ties together resolution and backend forwarding. //! -//! [`ProxyHandler`] is generic over the runtime's body type, backend client, -//! and request resolver. This allows it to be used identically on both -//! the server (Tokio/Hyper) and worker (Cloudflare Workers) runtimes. +//! [`ProxyHandler`] is generic over the runtime's backend and request resolver. +//! GET/HEAD/PUT/LIST operations go through `object_store`; multipart operations +//! use raw signed HTTP requests. -use crate::backend::{BackendClient, BackendRequest, S3RequestSigner, UNSIGNED_PAYLOAD}; +use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLOAD}; use crate::error::ProxyError; -use crate::resolver::{ResolvedAction, RequestResolver}; -use crate::s3::list_rewrite; +use crate::resolver::{ListRewrite, ResolvedAction, RequestResolver}; +use crate::response_body::ProxyResponseBody; use crate::s3::response::ErrorResponse; -use crate::stream::BodyStream; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; +use object_store::{GetOptions, GetRange, ObjectStore, PutPayload}; use url::Url; use uuid::Uuid; @@ -20,20 +20,20 @@ use uuid::Uuid; /// /// # Type Parameters /// -/// - `C`: The backend HTTP client for outbound requests to the backing store +/// - `B`: The runtime's backend for object store creation and raw HTTP /// - `R`: The request resolver that decides what action to take for each request -pub struct ProxyHandler { - client: C, +pub struct ProxyHandler { + backend: B, resolver: R, } -impl ProxyHandler +impl ProxyHandler where - C: BackendClient, + B: ProxyBackend, R: RequestResolver, { - pub fn new(client: C, resolver: R) -> Self { - Self { client, resolver } + pub fn new(backend: B, resolver: R) -> Self { + Self { backend, resolver } } /// Handle an incoming S3 request. @@ -41,15 +41,14 @@ where /// This is the main entry point. It: /// 1. Resolves the request via the resolver (parse, auth, authorize) /// 2. Forwards the request to the backing store or returns a synthetic response - /// 3. Optionally rewrites list response XML pub async fn handle_request( &self, method: Method, path: &str, query: Option<&str>, headers: &HeaderMap, - body: C::Body, - ) -> ProxyResult { + body: Bytes, + ) -> ProxyResult { let request_id = Uuid::new_v4().to_string(); tracing::info!( @@ -91,8 +90,8 @@ where path: &str, query: Option<&str>, headers: &HeaderMap, - body: C::Body, - ) -> Result, ProxyError> { + body: Bytes, + ) -> Result { let action = self.resolver.resolve(&method, path, query, headers).await?; match action { @@ -103,15 +102,22 @@ where } => Ok(ProxyResult { status, headers: resp_headers, - body: C::Body::from_bytes(resp_body), + body: ProxyResponseBody::from_bytes(resp_body), }), ResolvedAction::Proxy { operation, bucket_config, list_rewrite, } => { - self.forward_to_backend(&method, &operation, &bucket_config, headers, body, list_rewrite.as_ref()) - .await + self.forward_to_backend( + &method, + &operation, + &bucket_config, + headers, + body, + list_rewrite.as_ref(), + ) + .await } } } @@ -122,13 +128,278 @@ where operation: &S3Operation, bucket_config: &BucketConfig, original_headers: &HeaderMap, - body: C::Body, - list_rewrite: Option<&crate::resolver::ListRewrite>, - ) -> Result, ProxyError> { - // Build the backend URL + body: Bytes, + list_rewrite: Option<&ListRewrite>, + ) -> Result { + match operation { + S3Operation::GetObject { key, .. } => { + self.handle_get(bucket_config, key, original_headers).await + } + S3Operation::HeadObject { key, .. } => { + self.handle_head(bucket_config, key).await + } + S3Operation::PutObject { key, .. } => { + self.handle_put(bucket_config, key, body).await + } + S3Operation::ListBucket { raw_query, .. } => { + self.handle_list(bucket_config, raw_query.as_deref(), list_rewrite) + .await + } + // Multipart operations go through raw signed HTTP + S3Operation::CreateMultipartUpload { .. } + | S3Operation::UploadPart { .. } + | S3Operation::CompleteMultipartUpload { .. } + | S3Operation::AbortMultipartUpload { .. } => { + self.handle_multipart(method, operation, bucket_config, original_headers, body) + .await + } + _ => Err(ProxyError::Internal("unexpected operation".into())), + } + } + + /// GET via object_store + async fn handle_get( + &self, + config: &BucketConfig, + key: &str, + headers: &HeaderMap, + ) -> Result { + let store = self.backend.create_store(config)?; + let path = build_object_path(config, key); + + let mut opts = GetOptions::default(); + + // Parse conditional headers + if let Some(val) = headers.get("if-match").and_then(|v| v.to_str().ok()) { + opts.if_match = Some(val.to_string()); + } + if let Some(val) = headers.get("if-none-match").and_then(|v| v.to_str().ok()) { + opts.if_none_match = Some(val.to_string()); + } + if let Some(val) = headers.get("if-modified-since").and_then(|v| v.to_str().ok()) { + if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(val) { + opts.if_modified_since = Some(dt.with_timezone(&chrono::Utc)); + } + } + if let Some(val) = headers + .get("if-unmodified-since") + .and_then(|v| v.to_str().ok()) + { + if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(val) { + opts.if_unmodified_since = Some(dt.with_timezone(&chrono::Utc)); + } + } + + // Parse Range header + if let Some(range_val) = headers.get("range").and_then(|v| v.to_str().ok()) { + if let Some(range) = parse_range_header(range_val) { + opts.range = Some(range); + } + } + + tracing::debug!(path = %path, "GET via object_store"); + + let result = store + .get_opts(&path, opts) + .await + .map_err(ProxyError::from_object_store_error)?; + + // Build response headers from metadata + let mut resp_headers = HeaderMap::new(); + if let Some(etag) = &result.meta.e_tag { + resp_headers.insert("etag", etag.parse().unwrap()); + } + resp_headers.insert( + "last-modified", + result + .meta + .last_modified + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string() + .parse() + .unwrap(), + ); + let content_length = result.range.end - result.range.start; + resp_headers.insert("content-length", content_length.to_string().parse().unwrap()); + resp_headers.insert("accept-ranges", "bytes".parse().unwrap()); + + // If this is a range response, set 206 + Content-Range + let status = if result.range.start > 0 + || result.range.end < result.meta.size + { + resp_headers.insert( + "content-range", + format!( + "bytes {}-{}/{}", + result.range.start, + result.range.end.saturating_sub(1), + result.meta.size + ) + .parse() + .unwrap(), + ); + 206 + } else { + 200 + }; + + let stream = result.into_stream(); + + Ok(ProxyResult { + status, + headers: resp_headers, + body: ProxyResponseBody::Stream(stream), + }) + } + + /// HEAD via object_store + async fn handle_head( + &self, + config: &BucketConfig, + key: &str, + ) -> Result { + let store = self.backend.create_store(config)?; + let path = build_object_path(config, key); + + tracing::debug!(path = %path, "HEAD via object_store"); + + let meta = store + .head(&path) + .await + .map_err(ProxyError::from_object_store_error)?; + + let mut resp_headers = HeaderMap::new(); + if let Some(etag) = &meta.e_tag { + resp_headers.insert("etag", etag.parse().unwrap()); + } + resp_headers.insert( + "last-modified", + meta.last_modified + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string() + .parse() + .unwrap(), + ); + resp_headers.insert("content-length", meta.size.to_string().parse().unwrap()); + resp_headers.insert("accept-ranges", "bytes".parse().unwrap()); + + Ok(ProxyResult { + status: 200, + headers: resp_headers, + body: ProxyResponseBody::Empty, + }) + } + + /// PUT via object_store + async fn handle_put( + &self, + config: &BucketConfig, + key: &str, + body: Bytes, + ) -> Result { + let store = self.backend.create_store(config)?; + let path = build_object_path(config, key); + + tracing::debug!(path = %path, body_len = body.len(), "PUT via object_store"); + + let payload = PutPayload::from(body); + let result = store + .put(&path, payload) + .await + .map_err(ProxyError::from_object_store_error)?; + + let mut resp_headers = HeaderMap::new(); + if let Some(etag) = &result.e_tag { + resp_headers.insert("etag", etag.parse().unwrap()); + } + + Ok(ProxyResult { + status: 200, + headers: resp_headers, + body: ProxyResponseBody::Empty, + }) + } + + /// LIST via object_store + async fn handle_list( + &self, + config: &BucketConfig, + raw_query: Option<&str>, + list_rewrite: Option<&ListRewrite>, + ) -> Result { + let store = self.backend.create_store(config)?; + + // Extract prefix from query string + let client_prefix = raw_query + .and_then(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "prefix") + .map(|(_, v)| v.to_string()) + }) + .unwrap_or_default(); + + // Extract delimiter from query string (default "/") + let delimiter = raw_query + .and_then(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "delimiter") + .map(|(_, v)| v.to_string()) + }) + .unwrap_or_else(|| "/".to_string()); + + // Build the full prefix including backend_prefix + let full_prefix = build_list_prefix(config, &client_prefix); + + tracing::debug!( + full_prefix = %full_prefix, + delimiter = %delimiter, + "LIST via object_store" + ); + + let prefix_path = if full_prefix.is_empty() { + None + } else { + Some(object_store::path::Path::from(full_prefix.as_str())) + }; + + let list_result = store + .list_with_delimiter(prefix_path.as_ref()) + .await + .map_err(ProxyError::from_object_store_error)?; + + // Build S3 XML response from ListResult + let bucket_name = &config.name; + let xml = build_list_xml( + bucket_name, + &client_prefix, + &delimiter, + &list_result, + config, + list_rewrite, + ); + + let mut resp_headers = HeaderMap::new(); + resp_headers.insert("content-type", "application/xml".parse().unwrap()); + + Ok(ProxyResult { + status: 200, + headers: resp_headers, + body: ProxyResponseBody::Bytes(Bytes::from(xml)), + }) + } + + /// Multipart operations via raw signed HTTP + async fn handle_multipart( + &self, + method: &Method, + operation: &S3Operation, + bucket_config: &BucketConfig, + original_headers: &HeaderMap, + body: Bytes, + ) -> Result { let backend_url = build_backend_url(bucket_config, operation)?; - tracing::debug!(backend_url = %backend_url, "forwarding request to backend"); + tracing::debug!(backend_url = %backend_url, "multipart via raw HTTP"); let mut headers = HeaderMap::new(); @@ -137,35 +408,33 @@ where "content-type", "content-length", "content-md5", - "range", - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", ] { if let Some(val) = original_headers.get(*header_name) { headers.insert(*header_name, val.clone()); } } - // Only sign the outbound request if the backend has credentials configured. - // Public backends (e.g. source.coop) don't need signing. + // Sign the request if credentials are configured let has_credentials = !bucket_config.backend_access_key_id.is_empty() && !bucket_config.backend_secret_access_key.is_empty(); let parsed_url = Url::parse(&backend_url) .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; + let payload_hash = if body.is_empty() { + UNSIGNED_PAYLOAD.to_string() + } else { + hash_payload(&body) + }; + if has_credentials { let signer = S3RequestSigner::new( bucket_config.backend_access_key_id.clone(), bucket_config.backend_secret_access_key.clone(), bucket_config.backend_region.clone(), ); - signer.sign_request(method, &parsed_url, &mut headers, UNSIGNED_PAYLOAD)?; - tracing::trace!("outbound request signed with SigV4"); + signer.sign_request(method, &parsed_url, &mut headers, &payload_hash)?; } else { - // For unsigned requests, still set the host header let host = parsed_url .host_str() .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; @@ -175,63 +444,33 @@ where host.to_string() }; headers.insert("host", host_header.parse().unwrap()); - tracing::trace!("outbound request unsigned (public backend)"); } - let backend_req = BackendRequest { - method: method.clone(), - url: backend_url, - headers, - body, - }; - - let backend_resp = self.client.send_request(backend_req).await?; + let raw_resp = self + .backend + .send_raw(method.clone(), backend_url, headers, body) + .await?; - tracing::debug!( - status = backend_resp.status, - "backend response received" - ); - - // Apply list rewrite if configured and this is a successful list response - if let Some(rewrite) = list_rewrite { - if matches!(operation, S3Operation::ListBucket { .. }) - && backend_resp.status >= 200 - && backend_resp.status < 300 - { - // List responses are small XML — safe to buffer - let body_bytes = backend_resp - .body - .read_to_bytes() - .await - .map_err(|e| ProxyError::Internal(format!("failed to read list response: {}", e)))?; - let xml_str = String::from_utf8_lossy(&body_bytes); - let rewritten = list_rewrite::rewrite_list_response(&xml_str, rewrite); - return Ok(ProxyResult { - status: backend_resp.status, - headers: backend_resp.headers, - body: C::Body::from_bytes(Bytes::from(rewritten)), - }); - } - } + tracing::debug!(status = raw_resp.status, "multipart backend response"); Ok(ProxyResult { - status: backend_resp.status, - headers: backend_resp.headers, - body: backend_resp.body, + status: raw_resp.status, + headers: raw_resp.headers, + body: ProxyResponseBody::from_bytes(raw_resp.body), }) } } /// The result of handling a proxy request. -pub struct ProxyResult { +pub struct ProxyResult { pub status: u16, pub headers: HeaderMap, - pub body: B, + pub body: ProxyResponseBody, } -fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { +fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { let xml = ErrorResponse::from_proxy_error(err, resource, request_id).to_xml(); - let body = B::from_bytes(Bytes::from(xml)); + let body = ProxyResponseBody::from_bytes(Bytes::from(xml)); let mut headers = HeaderMap::new(); headers.insert("content-type", "application/xml".parse().unwrap()); @@ -242,6 +481,155 @@ fn error_response(err: &ProxyError, resource: &str, request_id: & } } +/// Build an object_store Path from a bucket config and client-visible key. +fn build_object_path(config: &BucketConfig, key: &str) -> object_store::path::Path { + let mut full_key = String::new(); + if let Some(prefix) = &config.backend_prefix { + let p = prefix.trim_end_matches('/'); + if !p.is_empty() { + full_key.push_str(p); + full_key.push('/'); + } + } + full_key.push_str(key); + object_store::path::Path::from(full_key) +} + +/// Build the full list prefix including backend_prefix. +fn build_list_prefix(config: &BucketConfig, client_prefix: &str) -> String { + let mut full_prefix = String::new(); + if let Some(bp) = &config.backend_prefix { + let bp = bp.trim_end_matches('/'); + if !bp.is_empty() { + full_prefix.push_str(bp); + full_prefix.push('/'); + } + } + full_prefix.push_str(client_prefix); + full_prefix +} + +/// Build S3 ListObjectsV2 XML from an object_store ListResult. +fn build_list_xml( + bucket_name: &str, + client_prefix: &str, + delimiter: &str, + list_result: &object_store::ListResult, + config: &BucketConfig, + list_rewrite: Option<&ListRewrite>, +) -> String { + let backend_prefix = config + .backend_prefix + .as_deref() + .unwrap_or("") + .trim_end_matches('/'); + let strip_prefix = if backend_prefix.is_empty() { + String::new() + } else { + format!("{}/", backend_prefix) + }; + + let mut xml = format!( + "\ + \ + {}\ + {}\ + {}\ + 1000\ + false\ + {}", + bucket_name, + client_prefix, + delimiter, + list_result.objects.len() + list_result.common_prefixes.len() + ); + + for obj in &list_result.objects { + let raw_key = obj.location.to_string(); + let key = rewrite_key(&raw_key, &strip_prefix, list_rewrite); + + let last_modified = obj.last_modified.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let etag = obj.e_tag.as_deref().unwrap_or("\"\""); + + xml.push_str(&format!( + "\ + {}\ + {}\ + {}\ + {}\ + STANDARD\ + ", + key, last_modified, etag, obj.size + )); + } + + for prefix_path in &list_result.common_prefixes { + let raw_prefix = format!("{}/", prefix_path); + let prefix = rewrite_key(&raw_prefix, &strip_prefix, list_rewrite); + + xml.push_str(&format!( + "{}", + prefix + )); + } + + xml.push_str(""); + xml +} + +/// Apply strip/add prefix rewriting to a key or prefix value. +fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite>) -> String { + let mut key = raw.to_string(); + + // Strip the backend prefix + if !strip_prefix.is_empty() { + if let Some(stripped) = key.strip_prefix(strip_prefix) { + key = stripped.to_string(); + } + } + + // Apply list_rewrite if present + if let Some(rewrite) = list_rewrite { + if !rewrite.strip_prefix.is_empty() { + if let Some(stripped) = key.strip_prefix(&rewrite.strip_prefix) { + key = stripped.to_string(); + } + } + if !rewrite.add_prefix.is_empty() { + if key.is_empty() || key.starts_with('/') { + key = format!("{}{}", rewrite.add_prefix, key); + } else { + key = format!("{}/{}", rewrite.add_prefix, key); + } + } + } + + key +} + +/// Parse an HTTP Range header value into an object_store GetRange. +fn parse_range_header(value: &str) -> Option { + let range_str = value.strip_prefix("bytes=")?; + + if let Some(suffix) = range_str.strip_prefix('-') { + // bytes=-N (suffix) + let n: u64 = suffix.parse().ok()?; + return Some(GetRange::Suffix(n)); + } + + let (start_str, end_str) = range_str.split_once('-')?; + let start: u64 = start_str.parse().ok()?; + + if end_str.is_empty() { + // bytes=N- (offset to end) + Some(GetRange::Offset(start)) + } else { + // bytes=N-M (bounded, HTTP is inclusive, object_store is exclusive) + let end: u64 = end_str.parse().ok()?; + Some(GetRange::Bounded(start..end + 1)) + } +} + fn build_backend_url( config: &BucketConfig, operation: &S3Operation, @@ -251,10 +639,7 @@ fn build_backend_url( let bucket_is_empty = bucket.is_empty(); let key = match operation { - S3Operation::GetObject { key, .. } - | S3Operation::HeadObject { key, .. } - | S3Operation::PutObject { key, .. } - | S3Operation::CreateMultipartUpload { key, .. } + S3Operation::CreateMultipartUpload { key, .. } | S3Operation::UploadPart { key, .. } | S3Operation::CompleteMultipartUpload { key, .. } | S3Operation::AbortMultipartUpload { key, .. } => { @@ -266,23 +651,9 @@ fn build_backend_url( full_key.push_str(key); full_key } - S3Operation::ListBucket { raw_query, .. } => { - let base_url = if bucket_is_empty { - base.to_string() - } else { - format!("{}/{}", base, bucket) - }; - let query_string = build_list_query_string(raw_query.as_deref(), config); - if query_string.is_empty() { - return Ok(base_url); - } - return Ok(format!("{}?{}", base_url, query_string)); - } - _ => return Err(ProxyError::Internal("unexpected operation".into())), + _ => return Err(ProxyError::Internal("unexpected operation for multipart URL".into())), }; - // Build URL: skip bucket segment when backend_bucket is empty (e.g. source.coop - // where the endpoint itself is the bucket root) let mut url = if bucket_is_empty { format!("{}/{}", base, key) } else { @@ -309,49 +680,3 @@ fn build_backend_url( Ok(url) } - -/// Build the query string for a ListBucket backend request. -/// -/// - Forwards all incoming query params verbatim -/// - Prepends `backend_prefix` to the `prefix` param if configured -/// - Injects `list-type=2` if not specified (default to ListObjectsV2) -/// - Injects `max-keys=1000` if not specified -fn build_list_query_string(raw_query: Option<&str>, config: &BucketConfig) -> String { - let mut params: Vec<(String, String)> = raw_query - .map(|q| { - url::form_urlencoded::parse(q.as_bytes()) - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - }) - .unwrap_or_default(); - - // Merge backend_prefix into the prefix param - if let Some(backend_prefix) = &config.backend_prefix { - let bp = backend_prefix.trim_end_matches('/'); - if !bp.is_empty() { - if let Some((_k, v)) = params.iter_mut().find(|(k, _)| k == "prefix") { - // Prepend backend_prefix to the client-supplied prefix - *v = format!("{}/{}", bp, v); - } else { - // No client prefix — set prefix to the backend_prefix (with trailing /) - // so the list is scoped to the backend_prefix directory - params.push(("prefix".to_string(), format!("{}/", bp))); - } - } - } - - // Default to ListObjectsV2 if no list-type specified - if !params.iter().any(|(k, _)| k == "list-type") { - params.push(("list-type".to_string(), "2".to_string())); - } - - // Default max-keys to 1000 if not specified - if !params.iter().any(|(k, _)| k == "max-keys") { - params.push(("max-keys".to_string(), "1000".to_string())); - } - - // Re-encode - url::form_urlencoded::Serializer::new(String::new()) - .extend_pairs(params.iter()) - .finish() -} diff --git a/crates/libs/core/src/response_body.rs b/crates/libs/core/src/response_body.rs new file mode 100644 index 0000000..2eb3316 --- /dev/null +++ b/crates/libs/core/src/response_body.rs @@ -0,0 +1,37 @@ +//! Response body type for the proxy. +//! +//! [`ProxyResponseBody`] replaces the old generic body type parameter. +//! All runtimes convert this to their native response type at the edge. + +use bytes::Bytes; +use futures::stream::BoxStream; + +/// The body of a proxy response. +/// +/// This is no longer generic — all runtimes work with this concrete type +/// and convert to their native response format. +pub enum ProxyResponseBody { + /// Streaming response from `object_store` GET. + /// Bytes arrive lazily in chunks. + Stream(BoxStream<'static, Result>), + /// Fixed bytes (error XML, list XML, multipart XML responses, etc.). + Bytes(Bytes), + /// Empty body (HEAD responses, etc.). + Empty, +} + +impl ProxyResponseBody { + /// Create a response body from raw bytes. + pub fn from_bytes(bytes: Bytes) -> Self { + if bytes.is_empty() { + Self::Empty + } else { + Self::Bytes(bytes) + } + } + + /// Create an empty response body. + pub fn empty() -> Self { + Self::Empty + } +} diff --git a/crates/libs/core/src/stream.rs b/crates/libs/core/src/stream.rs deleted file mode 100644 index 54b2e86..0000000 --- a/crates/libs/core/src/stream.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Stream abstraction for runtime-agnostic body handling. -//! -//! The key insight: the core proxy logic almost never needs to inspect or -//! transform the bytes flowing through it. For GET/PUT, the body is opaque — -//! it comes in from one side and goes out the other. This means our trait -//! can be minimal: we just need to know a body type exists and can be passed -//! around. -//! -//! Each runtime provides its own concrete type: -//! - Server runtime: `hyper::body::Incoming` / `http_body_util::Full` -//! - Worker runtime: a wrapper around JS `ReadableStream` -//! -//! The only time the core reads body bytes is for `CompleteMultipartUpload` -//! (parsing the XML manifest), which uses the `read_to_bytes` method. - -use bytes::Bytes; -use std::future::Future; - -use crate::maybe_send::MaybeSend; - -/// Trait representing a streaming body type. -/// -/// This is intentionally minimal. The core passes bodies through opaquely; -/// it never iterates over chunks except when it must parse a small request body. -pub trait BodyStream: Sized + MaybeSend + 'static { - type Error: std::error::Error + Send + Sync + 'static; - - /// Consume the body and collect all bytes. - /// Used only for small bodies like XML manifests, never for large object data. - fn read_to_bytes(self) -> impl Future> + MaybeSend; - - /// Create a body from raw bytes. - fn from_bytes(bytes: Bytes) -> Self; - - /// Create an empty body. - fn empty() -> Self; - - /// Content length, if known. - fn content_length(&self) -> Option; -} diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index a616673..829a75e 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -20,6 +20,11 @@ thiserror.workspace = true chrono.workspace = true quick-xml.workspace = true url.workspace = true +object_store.workspace = true +futures.workspace = true +http-body.workspace = true +http-body-util.workspace = true +async-trait.workspace = true # Cloudflare Workers SDK worker = "0.7" @@ -35,4 +40,13 @@ web-sys = { version = "0.3", features = [ "ResponseInit", ] } console_error_panic_hook = "0.1.7" -getrandom = { version = "0.2", features = ["js"] } + +[dependencies.getrandom_v02] +package = "getrandom" +version = "0.2" +features = ["js"] + +[dependencies.getrandom_v03] +package = "getrandom" +version = "0.3" +features = ["wasm_js"] diff --git a/crates/runtimes/cf-workers/README.md b/crates/runtimes/cf-workers/README.md index 35ed42e..086af3f 100644 --- a/crates/runtimes/cf-workers/README.md +++ b/crates/runtimes/cf-workers/README.md @@ -1,6 +1,6 @@ # s3-proxy-cf-workers -Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, with JS `ReadableStream` passthrough — response bytes from the backing store never touch WASM memory. +Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, using `object_store` with a custom `FetchConnector` that bridges to the Workers Fetch API. ## How It Works @@ -12,19 +12,20 @@ Client request - SOURCE_API_URL set? -> SourceCoopResolver (dynamic Source Cooperative backends) - Otherwise -> DefaultResolver (static PROXY_CONFIG) -> ProxyHandler::handle_request() (from s3-proxy-core) - -> WorkerBackendClient sends fetch() to backend - -> Response returned to client (ReadableStream passthrough) + -> object_store (via FetchConnector) or raw fetch for multipart + -> ProxyResponseBody converted to worker::Response ``` -The `WorkerBackendClient` uses the Workers Fetch API for outbound requests. For GET responses, the JS `ReadableStream` from the backend is passed directly to the outbound `Response` — bytes never cross the WASM boundary. +`WorkerBackend` implements `ProxyBackend` using a custom `FetchConnector` that bridges `object_store` HTTP requests to the Workers Fetch API. Since JS interop types are `!Send`, `spawn_local` + channel patterns are used to bridge to the `Send` context that `object_store` expects. Response body streams are converted to JS `ReadableStream` via `TransformStream` for delivery to clients. ## Module Overview ``` src/ ├── lib.rs Worker entry point, request/response conversion (thin adapter) -├── body.rs WorkerBody implementing BodyStream (Bytes | ReadableStream | Empty) -├── client.rs WorkerBackendClient implementing BackendClient via Fetch API +├── body.rs ProxyResponseBody → worker::Response conversion +├── client.rs WorkerBackend implementing ProxyBackend, fetch_json helper +├── fetch_connector.rs FetchConnector/FetchService bridging object_store to Fetch API ├── source_api.rs HTTP client for the Source Cooperative API └── source_resolver.rs SourceCoopResolver implementing RequestResolver ``` @@ -93,7 +94,7 @@ Then add a branch in `lib.rs`: ```rust if let Ok(my_config) = env.var("MY_MODE") { let resolver = MyResolver::new(/* ... */); - let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + let handler = ProxyHandler::new(client::WorkerBackend, resolver); let result = handler.handle_request(method, &path, query.as_deref(), &headers, body).await; return build_worker_response(result); } diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs index 93b8d10..8d4fb42 100644 --- a/crates/runtimes/cf-workers/src/body.rs +++ b/crates/runtimes/cf-workers/src/body.rs @@ -1,122 +1,85 @@ -//! Worker body type implementing `BodyStream`. +//! Response body conversion for the Cloudflare Workers runtime. //! -//! The key optimization: response bodies from the backend Fetch API are -//! `ReadableStream` objects in JS. Rather than reading them into Rust memory, -//! we pass them through opaquely. The stream only touches Rust when the core -//! needs to parse a small body (e.g., CompleteMultipartUpload XML manifest). +//! Converts [`ProxyResponseBody`] to `worker::Response`. -use bytes::Bytes; +use futures::StreamExt; use js_sys::Uint8Array; -use s3_proxy_core::stream::BodyStream; -use wasm_bindgen_futures::JsFuture; +use s3_proxy_core::proxy::ProxyResult; +use s3_proxy_core::response_body::ProxyResponseBody; +use wasm_bindgen_futures::spawn_local; +use worker::{Headers, Response}; -/// Body type for the Cloudflare Workers runtime. +/// Build a `worker::Response` from a `ProxyResult`. /// -/// Most request/response bodies flow through as opaque JS `ReadableStream` -/// objects, never touching Rust memory. The `Bytes` variant is used only -/// for small bodies constructed in Rust (error responses, XML manifests). -pub enum WorkerBody { - /// Raw bytes (small bodies constructed in Rust). - Bytes(Bytes), - /// A JS ReadableStream passed through opaquely. - Stream(web_sys::ReadableStream), - /// No body. - Empty, -} - -#[derive(Debug, thiserror::Error)] -#[error("worker body error: {0}")] -pub struct WorkerBodyError(pub String); - -impl BodyStream for WorkerBody { - type Error = WorkerBodyError; - - async fn read_to_bytes(self) -> Result { - match self { - WorkerBody::Bytes(b) => Ok(b), - WorkerBody::Empty => Ok(Bytes::new()), - WorkerBody::Stream(stream) => { - // Consume the ReadableStream into bytes. - // This is only called for small bodies (XML manifests), never for - // large object data. - read_stream_to_bytes(stream).await - } +/// Stream bodies are bridged to JS `ReadableStream` via a `TransformStream`: +/// a spawn_local task reads Rust stream chunks and writes them to the +/// writable side; the readable side is used for the Response. +pub fn build_worker_response( + result: ProxyResult, +) -> Result { + let resp_headers = Headers::new(); + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = resp_headers.set(key.as_str(), v); } } - fn from_bytes(bytes: Bytes) -> Self { - if bytes.is_empty() { - WorkerBody::Empty - } else { - WorkerBody::Bytes(bytes) - } - } + match result.body { + ProxyResponseBody::Stream(stream) => { + // Bridge Rust Stream -> JS ReadableStream via TransformStream + let transform = web_sys::TransformStream::new() + .map_err(|e| worker::Error::RustError(format!("TransformStream error: {:?}", e)))?; - fn empty() -> Self { - WorkerBody::Empty - } + let writable = transform.writable(); + let readable = transform.readable(); - fn content_length(&self) -> Option { - match self { - WorkerBody::Bytes(b) => Some(b.len() as u64), - WorkerBody::Empty => Some(0), - // Stream length is unknown — the backend will set Content-Length - // in the response headers if applicable. - WorkerBody::Stream(_) => None, - } - } -} + // Spawn a task to pump chunks from the Rust stream into the JS writable side + spawn_local(async move { + let writer = writable.get_writer().unwrap(); + let mut stream = stream; + + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(bytes) => { + let uint8 = Uint8Array::from(bytes.as_ref()); + if let Err(_) = + wasm_bindgen_futures::JsFuture::from(writer.write_with_chunk(&uint8.into())).await + { + break; + } + } + Err(_) => break, + } + } + let _ = wasm_bindgen_futures::JsFuture::from(writer.close()).await; + }); -impl WorkerBody { - /// Convert to a JsValue for use as a Fetch API request body. - /// Returns `None` for empty bodies (Fetch API interprets absent body as no body). - pub fn into_js_body(self) -> Option { - match self { - WorkerBody::Empty => None, - WorkerBody::Bytes(b) => { - let uint8 = Uint8Array::from(b.as_ref()); - Some(uint8.into()) + // Build the response from the readable side + let ws_headers = web_sys::Headers::new() + .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } } - WorkerBody::Stream(stream) => Some(stream.into()), - } - } - /// Create a `WorkerBody` from a `web_sys::Response` by extracting its - /// ReadableStream body without consuming it into bytes. - pub fn from_ws_response(response: &web_sys::Response) -> Self { - match response.body() { - Some(stream) => WorkerBody::Stream(stream), - None => WorkerBody::Empty, - } - } + let init = web_sys::ResponseInit::new(); + init.set_status(result.status); + init.set_headers(&ws_headers.into()); + + let ws_response = + web_sys::Response::new_with_opt_readable_stream_and_init(Some(&readable), &init) + .map_err(|e| { + worker::Error::RustError(format!("failed to build response: {:?}", e)) + })?; - /// Create a `WorkerBody` from a `web_sys::Request` by extracting its - /// ReadableStream body. - pub fn from_ws_request(request: &web_sys::Request) -> Self { - match request.body() { - Some(stream) => WorkerBody::Stream(stream), - None => WorkerBody::Empty, + Ok(ws_response.into()) } + ProxyResponseBody::Bytes(b) => Ok(Response::from_bytes(b.to_vec())? + .with_status(result.status) + .with_headers(resp_headers)), + ProxyResponseBody::Empty => Ok(Response::from_bytes(vec![])? + .with_status(result.status) + .with_headers(resp_headers)), } } - -/// Read a JS ReadableStream to completion, collecting all chunks into `Bytes`. -/// -/// Uses the JS `Response` constructor trick: `new Response(stream).arrayBuffer()` -/// which is the most efficient way to consume a stream in Workers. -async fn read_stream_to_bytes(stream: web_sys::ReadableStream) -> Result { - // Create a Response from the stream, then read its arrayBuffer - let response = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) - .map_err(|e| WorkerBodyError(format!("failed to wrap stream in Response: {:?}", e)))?; - - let array_buffer_promise = response - .array_buffer() - .map_err(|e| WorkerBodyError(format!("failed to get arrayBuffer: {:?}", e)))?; - - let array_buffer = JsFuture::from(array_buffer_promise) - .await - .map_err(|e| WorkerBodyError(format!("failed to read arrayBuffer: {:?}", e)))?; - - let uint8 = Uint8Array::new(&array_buffer); - Ok(Bytes::from(uint8.to_vec())) -} diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index a66807e..9e76714 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -1,13 +1,19 @@ -//! Backend client using the Cloudflare Workers Fetch API. +//! Backend client and HTTP helpers for the Cloudflare Workers runtime. //! -//! Uses `worker::Fetch` for subrequests. Response bodies for proxied S3 -//! requests are extracted as `ReadableStream` via `web_sys` for zero-copy -//! passthrough — bytes never enter Rust memory. - -use crate::body::WorkerBody; -use s3_proxy_core::backend::{BackendClient, BackendRequest, BackendResponse}; +//! Contains: +//! - `WorkerBackend` — implements `ProxyBackend` using the Fetch API + FetchConnector +//! - `fetch_json` — helper for server-to-server API calls (used by `source_api`) + +use crate::fetch_connector::FetchConnector; +use bytes::Bytes; +use http::HeaderMap; +use object_store::aws::AmazonS3Builder; +use object_store::ObjectStore; +use s3_proxy_core::backend::{ProxyBackend, RawResponse}; use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::BucketConfig; use serde::de::DeserializeOwned; +use std::sync::Arc; use worker::{Cache, Fetch}; /// Options for Cache API caching. @@ -105,70 +111,92 @@ pub(crate) async fn fetch_json( .map_err(|e| ProxyError::Internal(format!("failed to deserialize response: {}", e))) } -/// Backend client that uses the Workers Fetch API. +/// Backend for the Cloudflare Workers runtime. /// -/// Response bodies remain as opaque JS ReadableStreams — bytes never touch -/// Rust memory for passthrough requests (GET, PUT, etc.). -pub struct WorkerBackendClient; +/// Uses `FetchConnector` for `object_store` HTTP requests and `web_sys::fetch` +/// for raw multipart operations. +#[derive(Clone)] +pub struct WorkerBackend; + +impl ProxyBackend for WorkerBackend { + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + let mut builder = AmazonS3Builder::new() + .with_endpoint(&config.backend_endpoint) + .with_bucket_name(&config.backend_bucket) + .with_region(&config.backend_region) + .with_http_connector(FetchConnector); + + if !config.backend_access_key_id.is_empty() { + builder = builder + .with_access_key_id(&config.backend_access_key_id) + .with_secret_access_key(&config.backend_secret_access_key); + } else { + builder = builder.with_skip_signature(true); + } -impl BackendClient for WorkerBackendClient { - type Body = WorkerBody; + Ok(Arc::new( + builder + .build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, + )) + } - async fn send_request( + async fn send_raw( &self, - request: BackendRequest, - ) -> Result, ProxyError> { + method: http::Method, + url: String, + headers: HeaderMap, + body: Bytes, + ) -> Result { tracing::debug!( - method = %request.method, - url = %request.url, - "worker: sending backend request via Fetch API" + method = %method, + url = %url, + "worker: sending raw backend request via Fetch API" ); - // Build web_sys::Headers directly + // Build web_sys::Headers let ws_headers = web_sys::Headers::new() .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; - for (key, value) in request.headers.iter() { + for (key, value) in headers.iter() { if let Ok(v) = value.to_str() { let _ = ws_headers.set(key.as_str(), v); } } - // Build web_sys::RequestInit — we use web_sys types here because - // WorkerBody needs to pass JS ReadableStream/Uint8Array as the body. + // Build web_sys::RequestInit let init = web_sys::RequestInit::new(); - init.set_method(request.method.as_str()); + init.set_method(method.as_str()); init.set_headers(&ws_headers.into()); - // Set body for methods that carry one. - // Pass streams and bytes through as JS values — no materialization. - if matches!(request.method, http::Method::PUT | http::Method::POST) { - if let Some(js_body) = request.body.into_js_body() { - init.set_body(&js_body); - } + // Set body for methods that carry one + if !body.is_empty() { + let uint8 = js_sys::Uint8Array::from(body.as_ref()); + init.set_body(&uint8.into()); } let ws_request = - web_sys::Request::new_with_str_and_init(&request.url, &init).map_err(|e| { - tracing::error!(error = ?e, "failed to create web_sys::Request"); + web_sys::Request::new_with_str_and_init(&url, &init).map_err(|e| { ProxyError::BackendError(format!("failed to create request: {:?}", e)) })?; - // Convert to worker::Request and fetch via worker::Fetch. + // Fetch via worker let worker_req: worker::Request = ws_request.into(); - let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { - tracing::error!(url = %request.url, error = %e, "fetch to backend failed"); + let mut worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { ProxyError::BackendError(format!("fetch failed: {}", e)) })?; let status = worker_resp.status_code(); - tracing::debug!(status = status, "worker: backend response received"); - // Convert back to web_sys::Response to extract ReadableStream body. - let ws_response: web_sys::Response = worker_resp.into(); + // Read response body as bytes (multipart responses are small) + let resp_bytes = worker_resp + .bytes() + .await + .map_err(|e| ProxyError::Internal(format!("failed to read response: {}", e)))?; // Convert response headers - let mut resp_headers = http::HeaderMap::new(); + let ws_response: web_sys::Response = worker_resp.into(); + let mut resp_headers = HeaderMap::new(); let response_headers = ws_response.headers(); for name in &[ "content-type", @@ -177,8 +205,7 @@ impl BackendClient for WorkerBackendClient { "last-modified", "x-amz-request-id", "x-amz-version-id", - "accept-ranges", - "content-range", + "location", ] { if let Ok(Some(value)) = response_headers.get(name) { if let Ok(parsed) = value.parse() { @@ -187,13 +214,10 @@ impl BackendClient for WorkerBackendClient { } } - // Extract response body as a ReadableStream — zero-copy passthrough. - let body = WorkerBody::from_ws_response(&ws_response); - - Ok(BackendResponse { + Ok(RawResponse { status, headers: resp_headers, - body, + body: Bytes::from(resp_bytes), }) } } diff --git a/crates/runtimes/cf-workers/src/fetch_connector.rs b/crates/runtimes/cf-workers/src/fetch_connector.rs new file mode 100644 index 0000000..0f5790f --- /dev/null +++ b/crates/runtimes/cf-workers/src/fetch_connector.rs @@ -0,0 +1,162 @@ +//! Custom `HttpConnector` for `object_store` on Cloudflare Workers. +//! +//! Uses the Workers Fetch API to make HTTP requests, bridging the `!Send` +//! JS interop boundary via channels. +//! +//! Adapted from WITHOUT the stream +//! stashing hack — bytes flow through normally via mpsc channels. + +use bytes::Bytes; +use futures::channel::{mpsc, oneshot}; +use futures::{SinkExt, StreamExt}; +use http_body::Frame; +use http_body_util::StreamBody; +use object_store::client::{ + HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpRequest, HttpResponse, + HttpResponseBody, HttpService, +}; +use object_store::ClientOptions; +use wasm_bindgen_futures::spawn_local; + +/// A factory for creating HTTP clients that use the Workers Fetch API. +#[derive(Debug, Default, Clone)] +pub struct FetchConnector; + +impl HttpConnector for FetchConnector { + fn connect(&self, _options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(FetchService)) + } +} + +/// HTTP service implementation using the Workers Fetch API. +/// +/// Each `call()` spawns a `spawn_local` task because `worker::Fetch::send()` +/// returns a `!Send` future. A oneshot channel bridges the result back to +/// the `Send` context that `object_store` expects. +#[derive(Debug, Clone)] +struct FetchService; + +impl FetchService { + async fn do_fetch( + &self, + worker_req: worker::Request, + ) -> Result { + let (tx, rx) = oneshot::channel(); + + spawn_local(async move { + let result = Self::fetch_inner(worker_req).await; + let _ = tx.send(result); + }); + + rx.await.unwrap_or_else(|_| { + Err(HttpError::new( + HttpErrorKind::Unknown, + std::io::Error::new(std::io::ErrorKind::BrokenPipe, "fetch channel dropped"), + )) + }) + } + + async fn fetch_inner( + worker_req: worker::Request, + ) -> Result { + let mut resp = worker::Fetch::Request(worker_req) + .send() + .await + .map_err(|e| HttpError::new(HttpErrorKind::Unknown, e))?; + + let status = http::StatusCode::from_u16(resp.status_code()) + .unwrap_or(http::StatusCode::INTERNAL_SERVER_ERROR); + + // Convert response headers + let mut headers = http::HeaderMap::new(); + let worker_headers = resp.headers(); + for (key, value) in worker_headers.entries() { + if let (Ok(name), Ok(val)) = ( + http::header::HeaderName::try_from(key.as_str()), + http::header::HeaderValue::try_from(value.as_str()), + ) { + headers.insert(name, val); + } + } + + // Convert body: stream via mpsc channel + let body = match resp.stream() { + Ok(byte_stream) => byte_stream_to_http_body(byte_stream).await, + Err(_) => { + // Fall back to reading body as bytes + let body_bytes = resp + .bytes() + .await + .map_err(|e| HttpError::new(HttpErrorKind::Unknown, e))?; + HttpResponseBody::from(Bytes::from(body_bytes)) + } + }; + + let mut http_response = HttpResponse::new(body); + *http_response.status_mut() = status; + *http_response.headers_mut() = headers; + + Ok(http_response) + } +} + +#[async_trait::async_trait] +impl HttpService for FetchService { + async fn call(&self, req: HttpRequest) -> Result { + // Convert http::Request to worker::Request + let method = req.method().to_string(); + let uri = req.uri().to_string(); + let headers = req.headers().clone(); + + let mut worker_req = worker::Request::new(&uri, worker::Method::from(method)) + .map_err(|e| HttpError::new(HttpErrorKind::Unknown, e))?; + + // Copy headers + { + let worker_headers = worker_req + .headers_mut() + .map_err(|e| HttpError::new(HttpErrorKind::Unknown, e))?; + for (key, value) in headers.iter() { + if let Ok(v) = value.to_str() { + let _ = worker_headers.set(key.as_str(), v); + } + } + } + + self.do_fetch(worker_req).await + } +} + +/// Convert a `worker::ByteStream` to an `HttpResponseBody` via mpsc channel. +/// +/// The ByteStream is consumed in a `spawn_local` task (non-Send context). +/// Chunks are sent through an mpsc channel whose receiver implements `Send`, +/// which is then wrapped as a streaming `HttpResponseBody`. +async fn byte_stream_to_http_body(mut stream: worker::ByteStream) -> HttpResponseBody { + let (mut tx, rx) = mpsc::channel(1); + + spawn_local(async move { + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + if tx.send(Ok(Bytes::from(bytes))).await.is_err() { + break; + } + } + Err(e) => { + let _ = tx + .send(Err(HttpError::new(HttpErrorKind::Unknown, e))) + .await; + break; + } + } + } + }); + + let framed = rx.map(|chunk| { + let frame = Frame::data(chunk?); + Ok(frame) + }); + + HttpResponseBody::new(StreamBody::new(framed)) +} diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 7ab0b7c..b57321f 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -1,22 +1,19 @@ //! Cloudflare Workers runtime for the S3 proxy gateway. //! //! This crate provides implementations of core traits using Cloudflare Workers -//! primitives. The key advantage: response bodies from backend object stores -//! remain as JS `ReadableStream` objects throughout the proxy pipeline, avoiding -//! any conversion to/from Rust byte streams. +//! primitives. Response bodies from `object_store` are bridged from Rust +//! `Stream` to JS `ReadableStream` via a `TransformStream`. //! //! # Architecture //! //! ```text //! Client -> Worker (JS Request) //! -> resolve request (core resolver or Source Cooperative resolver) -//! -> fetch from backend (JS Fetch API -> JS Response with ReadableStream body) -//! -> return JS Response with ReadableStream body directly +//! -> object_store operation (via FetchConnector -> Fetch API) +//! -> ProxyResponseBody::Stream -> TransformStream -> JS ReadableStream +//! -> return JS Response //! ``` //! -//! The body bytes never touch Rust memory for GET requests. This is the primary -//! performance advantage of the multi-runtime architecture. -//! //! # Configuration //! //! On Workers, configuration is loaded from: @@ -27,15 +24,15 @@ mod body; mod client; +mod fetch_connector; mod source_api; mod source_resolver; mod tracing_layer; -use body::WorkerBody; +use body::build_worker_response; use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; use s3_proxy_core::proxy::ProxyHandler; use s3_proxy_core::resolver::DefaultResolver; -use s3_proxy_core::stream::BodyStream; use worker::*; /// The Worker entry point. @@ -58,11 +55,11 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let query = url.query().map(|q| q.to_string()); let headers = convert_headers(&req); - // Extract the request body as a JS ReadableStream — zero CPU cost. + // Materialize request body to Bytes for PUT/POST, empty for others let body = if matches!(method, http::Method::PUT | http::Method::POST) { - WorkerBody::from_ws_request(req.inner()) + read_request_body(&req).await? } else { - WorkerBody::empty() + bytes::Bytes::new() }; // Source Cooperative API mode: when SOURCE_API_URL is set, resolve backends @@ -108,7 +105,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let api_client = source_api::SourceApiClient::new(source_api_url.to_string(), source_api_key, cache_ttls); let resolver = source_resolver::SourceCoopResolver::new(api_client); - let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + let handler = ProxyHandler::new(client::WorkerBackend, resolver); let result = handler .handle_request(method, &path, query.as_deref(), &headers, body) @@ -136,7 +133,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); let resolver = DefaultResolver::new(config, virtual_host_domain); - let handler = ProxyHandler::new(client::WorkerBackendClient, resolver); + let handler = ProxyHandler::new(client::WorkerBackend, resolver); let result = handler .handle_request(method, &path, query.as_deref(), &headers, body) @@ -147,54 +144,27 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { // ── Shared helpers ────────────────────────────────────────────────── -/// Build a `worker::Response` from a `ProxyResult`, preserving stream bodies. -fn build_worker_response( - result: s3_proxy_core::proxy::ProxyResult, -) -> Result { - match result.body { - WorkerBody::Stream(stream) => { - let ws_headers = web_sys::Headers::new() - .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } - } - - let init = web_sys::ResponseInit::new(); - init.set_status(result.status); - init.set_headers(&ws_headers.into()); - - let ws_response = - web_sys::Response::new_with_opt_readable_stream_and_init(Some(&stream), &init) - .map_err(|e| { - worker::Error::RustError(format!("failed to build response: {:?}", e)) - })?; - - Ok(ws_response.into()) - } - WorkerBody::Bytes(b) => { - let resp_headers = Headers::new(); - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = resp_headers.set(key.as_str(), v); - } - } - Ok(Response::from_bytes(b.to_vec())? - .with_status(result.status) - .with_headers(resp_headers)) - } - WorkerBody::Empty => { - let resp_headers = Headers::new(); - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = resp_headers.set(key.as_str(), v); - } - } - Ok(Response::from_bytes(vec![])? - .with_status(result.status) - .with_headers(resp_headers)) +/// Read a Worker request body into Bytes. +async fn read_request_body(req: &Request) -> Result { + // Extract body as ReadableStream, consume to bytes + let ws_request = req.inner(); + match ws_request.body() { + Some(stream) => { + let response = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) + .map_err(|e| worker::Error::RustError(format!("failed to wrap stream: {:?}", e)))?; + + let array_buffer_promise = response + .array_buffer() + .map_err(|e| worker::Error::RustError(format!("failed to get arrayBuffer: {:?}", e)))?; + + let array_buffer = wasm_bindgen_futures::JsFuture::from(array_buffer_promise) + .await + .map_err(|e| worker::Error::RustError(format!("failed to read arrayBuffer: {:?}", e)))?; + + let uint8 = js_sys::Uint8Array::new(&array_buffer); + Ok(bytes::Bytes::from(uint8.to_vec())) } + None => Ok(bytes::Bytes::new()), } } diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index 9b49a37..5add142 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -20,3 +20,5 @@ serde.workspace = true toml.workspace = true reqwest = { workspace = true, features = ["stream"] } thiserror.workspace = true +object_store.workspace = true +futures.workspace = true diff --git a/crates/runtimes/server/README.md b/crates/runtimes/server/README.md index e311d21..0c9dad5 100644 --- a/crates/runtimes/server/README.md +++ b/crates/runtimes/server/README.md @@ -4,11 +4,9 @@ Tokio/Hyper runtime for the S3 proxy gateway. This is the container-deployment c ## What This Crate Provides -Three concrete implementations of core traits, plus a server binary: +A `ProxyBackend` implementation plus a server binary: -**`ServerBody`** — implements `BodyStream` using `http-body-util`. Wraps `Full`, `Empty`, or a streaming `reqwest::Response` for backend responses. Backend response bodies remain as reqwest's streaming type until consumed, avoiding unnecessary buffering. - -**`ReqwestBackendClient`** — implements `BackendClient` using `reqwest`. Sends signed requests to backing object stores with connection pooling (`pool_max_idle_per_host = 20`). +**`ServerBackend`** — implements `ProxyBackend`. Uses `object_store` with its default HTTP connector for high-level operations (GET, HEAD, PUT, LIST) and `reqwest` for raw multipart requests. GET responses stream from `object_store` through Hyper without buffering. **`server::run()`** — starts a Hyper HTTP server that accepts connections and delegates to `ProxyHandler` with a `DefaultResolver`. Supports both path-style (`/bucket/key`) and virtual-hosted-style (`bucket.s3.example.com/key`) routing via the resolver's `virtual_host_domain` setting. @@ -17,8 +15,8 @@ Three concrete implementations of core traits, plus a server binary: ``` src/ ├── lib.rs Crate root -├── body.rs ServerBody implementing BodyStream -├── client.rs ReqwestBackendClient implementing BackendClient +├── body.rs ProxyResponseBody → Hyper streaming response conversion +├── client.rs ServerBackend implementing ProxyBackend ├── server.rs Hyper server setup, request routing └── bin/ └── s3-proxy.rs CLI binary entry point @@ -98,8 +96,8 @@ impl RequestResolver for MyResolver { } // Then create the handler directly: -let client = ReqwestBackendClient::new(); -let handler = ProxyHandler::new(client, MyResolver::new()); +let backend = ServerBackend::new(); +let handler = ProxyHandler::new(backend, MyResolver::new()); // Use handler.handle_request() in your Hyper service. ``` @@ -108,8 +106,10 @@ See `s3-proxy-cf-workers/src/source_resolver.rs` for a complete example. ## Streaming Behavior -For **GET/HEAD** responses, the backend response body stays as a `reqwest::Response` (streaming) and is forwarded to the client. The proxy does not buffer the full object in memory. +For **GET** responses, `object_store` returns a `BoxStream` which is bridged to a Hyper streaming response body. Bytes flow through without buffering the entire object in memory. + +For **HEAD** responses, only metadata is returned (empty body). -For **PUT** request bodies, the current implementation collects the incoming body before forwarding. A follow-up optimization would pipe the incoming Hyper body directly to the reqwest request body using `reqwest::Body::wrap_stream`. +For **PUT** request bodies, the incoming Hyper body is collected to `Bytes` before passing to `object_store::put()`. -For **multipart uploads**, each part is individually streamed. The `CompleteMultipartUpload` request body (a small XML manifest) is the only body the proxy fully reads and parses. +For **multipart uploads**, operations are sent as raw signed HTTP requests via `reqwest`. The `CompleteMultipartUpload` request body (a small XML manifest) is the only body the proxy fully reads and parses. diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs index 0f6bb0b..37b3bb8 100644 --- a/crates/runtimes/server/src/body.rs +++ b/crates/runtimes/server/src/body.rs @@ -1,63 +1,48 @@ -//! Server-side body type implementing `BodyStream`. +//! Response body conversion for the server runtime. +//! +//! Converts [`ProxyResponseBody`] to a streaming hyper response body. use bytes::Bytes; -use http_body_util::{BodyExt, Full, Empty}; -use s3_proxy_core::stream::BodyStream; - -/// A body type for the server runtime. +use futures::TryStreamExt; +use http::Response; +use http_body_util::{Either, Empty, Full, StreamBody}; +use hyper::body::Frame; +use s3_proxy_core::proxy::ProxyResult; +use s3_proxy_core::response_body::ProxyResponseBody; + +/// A boxed streaming body type that erases concrete stream types. +type BoxedStreamBody = StreamBody< + std::pin::Pin< + Box, std::io::Error>> + Send>, + >, +>; + +/// The server response body type: either a stream, fixed bytes, or empty. +pub type ServerResponseBody = Either, Empty>>; + +/// Convert a `ProxyResult` to a hyper `Response` with a streaming body. /// -/// Wraps either an incoming request body or a constructed response body. -/// Uses `http_body_util` types which integrate natively with Hyper. -pub enum ServerBody { - /// A body constructed from known bytes. - Full(Full), - /// An empty body. - Empty(Empty), - /// A streaming body from reqwest (for backend responses). - Streaming(reqwest::Response), -} - -/// Error type for server body operations. -#[derive(Debug, thiserror::Error)] -pub enum ServerBodyError { - #[error("hyper error: {0}")] - Hyper(String), - #[error("reqwest error: {0}")] - Reqwest(#[from] reqwest::Error), -} - -impl BodyStream for ServerBody { - type Error = ServerBodyError; - - async fn read_to_bytes(self) -> Result { - match self { - ServerBody::Full(full) => { - let collected = full.collect().await.map_err(|e| ServerBodyError::Hyper(e.to_string()))?; - Ok(collected.to_bytes()) - } - ServerBody::Empty(_) => Ok(Bytes::new()), - ServerBody::Streaming(resp) => { - resp.bytes().await.map_err(ServerBodyError::Reqwest) - } - } +/// This is an improvement over the old implementation: GET responses now +/// stream through without buffering the entire body in memory. +pub fn build_hyper_response( + result: ProxyResult, +) -> Result, Box> { + let mut builder = Response::builder().status(result.status); + + for (key, value) in result.headers.iter() { + builder = builder.header(key, value); } - fn from_bytes(bytes: Bytes) -> Self { - ServerBody::Full(Full::new(bytes)) - } - - fn empty() -> Self { - ServerBody::Empty(Empty::new()) - } - - fn content_length(&self) -> Option { - match self { - ServerBody::Full(f) => { - use hyper::body::Body; - f.size_hint().exact() - } - ServerBody::Empty(_) => Some(0), - ServerBody::Streaming(resp) => resp.content_length(), + let body = match result.body { + ProxyResponseBody::Stream(stream) => { + let framed = stream + .map_ok(Frame::data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); + Either::Left(StreamBody::new(Box::pin(framed) as std::pin::Pin, std::io::Error>> + Send>>)) } - } + ProxyResponseBody::Bytes(b) => Either::Right(Either::Left(Full::new(b))), + ProxyResponseBody::Empty => Either::Right(Either::Right(Empty::new())), + }; + + Ok(builder.body(body)?) } diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index ba45e3f..2a71903 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -1,19 +1,24 @@ -//! Backend client using reqwest for outbound HTTP requests. +//! Server backend using reqwest for raw HTTP and default object_store connector. -use crate::body::ServerBody; -use s3_proxy_core::backend::{BackendClient, BackendRequest, BackendResponse}; +use bytes::Bytes; +use http::HeaderMap; +use object_store::aws::AmazonS3Builder; +use object_store::ObjectStore; +use s3_proxy_core::backend::{ProxyBackend, RawResponse}; use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::BucketConfig; +use std::sync::Arc; -/// Backend client that uses `reqwest` to make outbound requests. +/// Backend for the Tokio/Hyper server runtime. /// -/// This keeps the response body as a `reqwest::Response` which can be -/// streamed back to the client without buffering. +/// Uses reqwest for raw HTTP (multipart operations) and the default +/// object_store HTTP connector for high-level operations. #[derive(Clone)] -pub struct ReqwestBackendClient { +pub struct ServerBackend { client: reqwest::Client, } -impl ReqwestBackendClient { +impl ServerBackend { pub fn new() -> Self { Self { client: reqwest::Client::builder() @@ -24,70 +29,72 @@ impl ReqwestBackendClient { } } -impl Default for ReqwestBackendClient { +impl Default for ServerBackend { fn default() -> Self { Self::new() } } -impl BackendClient for ReqwestBackendClient { - type Body = ServerBody; +impl ProxyBackend for ServerBackend { + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + let mut builder = AmazonS3Builder::new() + .with_endpoint(&config.backend_endpoint) + .with_bucket_name(&config.backend_bucket) + .with_region(&config.backend_region); - async fn send_request( + if !config.backend_access_key_id.is_empty() { + builder = builder + .with_access_key_id(&config.backend_access_key_id) + .with_secret_access_key(&config.backend_secret_access_key); + } else { + builder = builder.with_skip_signature(true); + } + + Ok(Arc::new( + builder + .build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, + )) + } + + async fn send_raw( &self, - request: BackendRequest, - ) -> Result, ProxyError> { + method: http::Method, + url: String, + headers: HeaderMap, + body: Bytes, + ) -> Result { tracing::debug!( - method = %request.method, - url = %request.url, - "server: sending backend request via reqwest" + method = %method, + url = %url, + "server: sending raw backend request via reqwest" ); - let mut req_builder = self.client.request(request.method, &request.url); + let mut req_builder = self.client.request(method, &url); - // Set headers - for (key, value) in request.headers.iter() { + for (key, value) in headers.iter() { req_builder = req_builder.header(key, value); } - // Set body - req_builder = match request.body { - ServerBody::Full(full) => { - use http_body_util::BodyExt; - let bytes = full - .collect() - .await - .map_err(|e| ProxyError::BackendError(e.to_string()))? - .to_bytes(); - req_builder.body(bytes) - } - ServerBody::Empty(_) => req_builder, - ServerBody::Streaming(resp) => { - let bytes = resp - .bytes() - .await - .map_err(|e| ProxyError::BackendError(e.to_string()))?; - req_builder.body(bytes) - } - }; + if !body.is_empty() { + req_builder = req_builder.body(body); + } - let response = req_builder - .send() - .await - .map_err(|e| { - tracing::error!(error = %e, "reqwest backend request failed"); - ProxyError::BackendError(e.to_string()) - })?; + let response = req_builder.send().await.map_err(|e| { + tracing::error!(error = %e, "reqwest raw request failed"); + ProxyError::BackendError(e.to_string()) + })?; let status = response.status().as_u16(); - let headers = response.headers().clone(); - - tracing::debug!(status = status, "server: backend response received"); + let resp_headers = response.headers().clone(); + let resp_body = response.bytes().await.map_err(|e| { + ProxyError::BackendError(format!("failed to read raw response body: {}", e)) + })?; - Ok(BackendResponse { + Ok(RawResponse { status, - headers, - body: ServerBody::Streaming(response), + headers: resp_headers, + body: resp_body, }) } } diff --git a/crates/runtimes/server/src/lib.rs b/crates/runtimes/server/src/lib.rs index 4672988..43930c8 100644 --- a/crates/runtimes/server/src/lib.rs +++ b/crates/runtimes/server/src/lib.rs @@ -3,8 +3,8 @@ //! This crate provides concrete implementations of the core traits for a //! standard server environment using Tokio and Hyper. //! -//! - [`body::ServerBody`] — implements `BodyStream` using `http-body-util` -//! - [`client::HyperBackendClient`] — implements `BackendClient` using `reqwest` +//! - [`client::ServerBackend`] — implements `ProxyBackend` using reqwest + object_store +//! - [`body`] — converts `ProxyResponseBody` to streaming hyper responses //! - [`server::run`] — starts the Hyper HTTP server pub mod body; diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 6f06ec9..70c0c29 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -1,11 +1,10 @@ //! HTTP server using Hyper, wiring everything together. -use crate::body::ServerBody; -use crate::client::ReqwestBackendClient; +use crate::body::{build_hyper_response, ServerResponseBody}; +use crate::client::ServerBackend; use bytes::Bytes; -use s3_proxy_core::stream::BodyStream; use http::{Request, Response}; -use http_body_util::{BodyExt, Full}; +use http_body_util::{BodyExt, Either, Full}; use hyper::body::Incoming; use hyper::service::service_fn; use hyper_util::rt::{TokioExecutor, TokioIo}; @@ -55,9 +54,9 @@ pub async fn run

(config: P, server_config: ServerConfig) -> Result<(), Box( req: Request, - handler: &ProxyHandler, -) -> Result>, Box> + handler: &ProxyHandler, +) -> Result, Box> where R: s3_proxy_core::resolver::RequestResolver + Send + Sync, { @@ -115,27 +114,12 @@ where let query = uri.query(); let headers = req.headers().clone(); - // Convert incoming body to ServerBody + // Materialize incoming body to Bytes let incoming_bytes = req.into_body().collect().await?.to_bytes(); - let body = ServerBody::from_bytes(incoming_bytes); let result = handler - .handle_request(method, path, query, &headers, body) + .handle_request(method, path, query, &headers, incoming_bytes) .await; - // Convert ProxyResult to hyper Response - let mut response = Response::builder().status(result.status); - - for (key, value) in result.headers.iter() { - response = response.header(key, value); - } - - // Get the response body bytes - let body_bytes = match result.body { - ServerBody::Streaming(resp) => resp.bytes().await.unwrap_or_default(), - ServerBody::Full(full) => full.collect().await.map(|c| c.to_bytes()).unwrap_or_default(), - ServerBody::Empty(_) => Bytes::new(), - }; - - Ok(response.body(Full::new(body_bytes))?) + build_hyper_response(result) } From 9c27f5c774781f7f5d77a4e7f291194bf5d0c975 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Feb 2026 22:14:25 -0800 Subject: [PATCH 03/82] Support delete object --- crates/libs/core/src/auth.rs | 4 +++- crates/libs/core/src/proxy.rs | 26 ++++++++++++++++++++++++++ crates/libs/core/src/resolver.rs | 3 ++- crates/libs/core/src/s3/request.rs | 2 ++ crates/libs/core/src/types.rs | 5 +++++ 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 5d26a2f..00ecffc 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -261,6 +261,7 @@ fn operation_to_action(op: &S3Operation) -> Action { S3Operation::UploadPart { .. } => Action::UploadPart, S3Operation::CompleteMultipartUpload { .. } => Action::CompleteMultipartUpload, S3Operation::AbortMultipartUpload { .. } => Action::AbortMultipartUpload, + S3Operation::DeleteObject { .. } => Action::DeleteObject, S3Operation::ListBuckets => Action::ListBucket, // Treated as a list operation S3Operation::AssumeRoleWithWebIdentity { .. } => Action::GetObject, // STS is handled separately } @@ -274,7 +275,8 @@ fn operation_bucket_key(op: &S3Operation) -> (String, String) { | S3Operation::CreateMultipartUpload { bucket, key } | S3Operation::UploadPart { bucket, key, .. } | S3Operation::CompleteMultipartUpload { bucket, key, .. } - | S3Operation::AbortMultipartUpload { bucket, key, .. } => { + | S3Operation::AbortMultipartUpload { bucket, key, .. } + | S3Operation::DeleteObject { bucket, key } => { (bucket.clone(), key.clone()) } S3Operation::ListBucket { bucket, raw_query } => { diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 2c90af3..9256aa1 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -141,6 +141,9 @@ where S3Operation::PutObject { key, .. } => { self.handle_put(bucket_config, key, body).await } + S3Operation::DeleteObject { key, .. } => { + self.handle_delete(bucket_config, key).await + } S3Operation::ListBucket { raw_query, .. } => { self.handle_list(bucket_config, raw_query.as_deref(), list_rewrite) .await @@ -320,6 +323,29 @@ where }) } + /// DELETE via object_store + async fn handle_delete( + &self, + config: &BucketConfig, + key: &str, + ) -> Result { + let store = self.backend.create_store(config)?; + let path = build_object_path(config, key); + + tracing::debug!(path = %path, "DELETE via object_store"); + + store + .delete(&path) + .await + .map_err(ProxyError::from_object_store_error)?; + + Ok(ProxyResult { + status: 204, + headers: HeaderMap::new(), + body: ProxyResponseBody::Empty, + }) + } + /// LIST via object_store async fn handle_list( &self, diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index 2d3db43..ae158b3 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -173,7 +173,8 @@ fn operation_bucket(op: &S3Operation) -> Option { | S3Operation::CreateMultipartUpload { bucket, .. } | S3Operation::UploadPart { bucket, .. } | S3Operation::CompleteMultipartUpload { bucket, .. } - | S3Operation::AbortMultipartUpload { bucket, .. } => Some(bucket.clone()), + | S3Operation::AbortMultipartUpload { bucket, .. } + | S3Operation::DeleteObject { bucket, .. } => Some(bucket.clone()), S3Operation::ListBuckets => None, S3Operation::AssumeRoleWithWebIdentity { .. } => None, } diff --git a/crates/libs/core/src/s3/request.rs b/crates/libs/core/src/s3/request.rs index 22833a2..d696de1 100644 --- a/crates/libs/core/src/s3/request.rs +++ b/crates/libs/core/src/s3/request.rs @@ -120,6 +120,8 @@ pub fn build_s3_operation( key, upload_id, }) + } else if !key.is_empty() { + Ok(S3Operation::DeleteObject { bucket, key }) } else { Err(ProxyError::InvalidRequest( "unsupported DELETE operation".into(), diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index 692bc97..fbe84ce 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -81,6 +81,7 @@ pub enum Action { UploadPart, CompleteMultipartUpload, AbortMultipartUpload, + DeleteObject, } /// A long-lived access credential stored in the config backend. @@ -155,6 +156,10 @@ pub enum S3Operation { key: String, upload_id: String, }, + DeleteObject { + bucket: String, + key: String, + }, ListBucket { bucket: String, /// Raw query string from the incoming request, forwarded to the backend. From e5b569832e8a1151049f5b2cea769e0b8336dfbe Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Feb 2026 22:37:55 -0800 Subject: [PATCH 04/82] Attempt to simplify by adding properties to S3Operation enum --- crates/libs/core/src/auth.rs | 64 +++++----------- crates/libs/core/src/resolver.rs | 22 +----- crates/libs/core/src/types.rs | 122 +++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 66 deletions(-) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 00ecffc..fd7131a 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -209,7 +209,7 @@ pub fn authorize( if matches!(identity, ResolvedIdentity::Anonymous) { if bucket_config.anonymous_access { // Anonymous users can only read - let action = operation_to_action(operation); + let action = operation.action(); if matches!(action, Action::GetObject | Action::HeadObject | Action::ListBucket) { return Ok(()); } @@ -223,8 +223,22 @@ pub fn authorize( ResolvedIdentity::Temporary { credentials } => &credentials.allowed_scopes, }; - let action = operation_to_action(operation); - let (bucket, key) = operation_bucket_key(operation); + let action = operation.action(); + let bucket = operation.bucket().unwrap_or_default().to_string(); + let key = match operation { + S3Operation::ListBucket { raw_query, .. } => { + // Extract prefix from raw query for authorization checks + raw_query + .as_deref() + .and_then(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "prefix") + .map(|(_, v)| v.to_string()) + }) + .unwrap_or_default() + } + _ => operation.key().to_string(), + }; // Check if any scope grants access let authorized = scopes.iter().any(|scope| { @@ -251,47 +265,3 @@ pub fn authorize( } } -fn operation_to_action(op: &S3Operation) -> Action { - match op { - S3Operation::GetObject { .. } => Action::GetObject, - S3Operation::HeadObject { .. } => Action::HeadObject, - S3Operation::PutObject { .. } => Action::PutObject, - S3Operation::ListBucket { .. } => Action::ListBucket, - S3Operation::CreateMultipartUpload { .. } => Action::CreateMultipartUpload, - S3Operation::UploadPart { .. } => Action::UploadPart, - S3Operation::CompleteMultipartUpload { .. } => Action::CompleteMultipartUpload, - S3Operation::AbortMultipartUpload { .. } => Action::AbortMultipartUpload, - S3Operation::DeleteObject { .. } => Action::DeleteObject, - S3Operation::ListBuckets => Action::ListBucket, // Treated as a list operation - S3Operation::AssumeRoleWithWebIdentity { .. } => Action::GetObject, // STS is handled separately - } -} - -fn operation_bucket_key(op: &S3Operation) -> (String, String) { - match op { - S3Operation::GetObject { bucket, key } - | S3Operation::HeadObject { bucket, key } - | S3Operation::PutObject { bucket, key } - | S3Operation::CreateMultipartUpload { bucket, key } - | S3Operation::UploadPart { bucket, key, .. } - | S3Operation::CompleteMultipartUpload { bucket, key, .. } - | S3Operation::AbortMultipartUpload { bucket, key, .. } - | S3Operation::DeleteObject { bucket, key } => { - (bucket.clone(), key.clone()) - } - S3Operation::ListBucket { bucket, raw_query } => { - // Extract prefix from raw query for authorization checks - let prefix = raw_query - .as_deref() - .and_then(|q| { - url::form_urlencoded::parse(q.as_bytes()) - .find(|(k, _)| k == "prefix") - .map(|(_, v)| v.to_string()) - }) - .unwrap_or_default(); - (bucket.clone(), prefix) - } - S3Operation::ListBuckets => (String::new(), String::new()), - S3Operation::AssumeRoleWithWebIdentity { .. } => (String::new(), String::new()), - } -} diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index ae158b3..834ae08 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -130,16 +130,16 @@ impl RequestResolver for DefaultResolver

{ } // Get bucket name and look up config - let bucket_name = operation_bucket(&operation) + let bucket_name = operation.bucket() .ok_or_else(|| ProxyError::InvalidRequest("no bucket in request".into()))?; let bucket_config = self .config - .get_bucket(&bucket_name) + .get_bucket(bucket_name) .await? .ok_or_else(|| { tracing::warn!(bucket = %bucket_name, "bucket not found in config"); - ProxyError::BucketNotFound(bucket_name.clone()) + ProxyError::BucketNotFound(bucket_name.to_string()) })?; tracing::debug!( @@ -164,22 +164,6 @@ impl RequestResolver for DefaultResolver

{ } } -fn operation_bucket(op: &S3Operation) -> Option { - match op { - S3Operation::GetObject { bucket, .. } - | S3Operation::HeadObject { bucket, .. } - | S3Operation::PutObject { bucket, .. } - | S3Operation::ListBucket { bucket, .. } - | S3Operation::CreateMultipartUpload { bucket, .. } - | S3Operation::UploadPart { bucket, .. } - | S3Operation::CompleteMultipartUpload { bucket, .. } - | S3Operation::AbortMultipartUpload { bucket, .. } - | S3Operation::DeleteObject { bucket, .. } => Some(bucket.clone()), - S3Operation::ListBuckets => None, - S3Operation::AssumeRoleWithWebIdentity { .. } => None, - } -} - fn determine_host_style(headers: &HeaderMap, virtual_host_domain: Option<&str>) -> HostStyle { if let Some(domain) = virtual_host_domain { if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) { diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index fbe84ce..fd804da 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -176,3 +176,125 @@ pub enum S3Operation { duration_seconds: Option, }, } + +impl S3Operation { + /// The authorization action for this operation. + pub fn action(&self) -> Action { + match self { + S3Operation::GetObject { .. } => Action::GetObject, + S3Operation::HeadObject { .. } => Action::HeadObject, + S3Operation::PutObject { .. } => Action::PutObject, + S3Operation::ListBucket { .. } => Action::ListBucket, + S3Operation::CreateMultipartUpload { .. } => Action::CreateMultipartUpload, + S3Operation::UploadPart { .. } => Action::UploadPart, + S3Operation::CompleteMultipartUpload { .. } => Action::CompleteMultipartUpload, + S3Operation::AbortMultipartUpload { .. } => Action::AbortMultipartUpload, + S3Operation::DeleteObject { .. } => Action::DeleteObject, + S3Operation::ListBuckets => Action::ListBucket, + S3Operation::AssumeRoleWithWebIdentity { .. } => Action::GetObject, // STS is handled separately + } + } + + /// The bucket name, if any. + pub fn bucket(&self) -> Option<&str> { + match self { + S3Operation::GetObject { bucket, .. } + | S3Operation::HeadObject { bucket, .. } + | S3Operation::PutObject { bucket, .. } + | S3Operation::ListBucket { bucket, .. } + | S3Operation::CreateMultipartUpload { bucket, .. } + | S3Operation::UploadPart { bucket, .. } + | S3Operation::CompleteMultipartUpload { bucket, .. } + | S3Operation::AbortMultipartUpload { bucket, .. } + | S3Operation::DeleteObject { bucket, .. } => Some(bucket), + S3Operation::ListBuckets => None, + S3Operation::AssumeRoleWithWebIdentity { .. } => None, + } + } + + /// The object key, if any. Returns empty string for non-object operations. + pub fn key(&self) -> &str { + match self { + S3Operation::GetObject { key, .. } + | S3Operation::HeadObject { key, .. } + | S3Operation::PutObject { key, .. } + | S3Operation::CreateMultipartUpload { key, .. } + | S3Operation::UploadPart { key, .. } + | S3Operation::CompleteMultipartUpload { key, .. } + | S3Operation::AbortMultipartUpload { key, .. } + | S3Operation::DeleteObject { key, .. } => key, + S3Operation::ListBucket { .. } + | S3Operation::ListBuckets + | S3Operation::AssumeRoleWithWebIdentity { .. } => "", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_action() { + let op = S3Operation::GetObject { + bucket: "b".into(), + key: "k".into(), + }; + assert_eq!(op.action(), Action::GetObject); + + let op = S3Operation::PutObject { + bucket: "b".into(), + key: "k".into(), + }; + assert_eq!(op.action(), Action::PutObject); + + let op = S3Operation::ListBucket { + bucket: "b".into(), + raw_query: None, + }; + assert_eq!(op.action(), Action::ListBucket); + + assert_eq!(S3Operation::ListBuckets.action(), Action::ListBucket); + + let op = S3Operation::DeleteObject { + bucket: "b".into(), + key: "k".into(), + }; + assert_eq!(op.action(), Action::DeleteObject); + } + + #[test] + fn test_bucket() { + let op = S3Operation::GetObject { + bucket: "my-bucket".into(), + key: "k".into(), + }; + assert_eq!(op.bucket(), Some("my-bucket")); + + assert_eq!(S3Operation::ListBuckets.bucket(), None); + + let op = S3Operation::AssumeRoleWithWebIdentity { + role_arn: "arn".into(), + web_identity_token: "tok".into(), + duration_seconds: None, + }; + assert_eq!(op.bucket(), None); + } + + #[test] + fn test_key() { + let op = S3Operation::GetObject { + bucket: "b".into(), + key: "my/key.txt".into(), + }; + assert_eq!(op.key(), "my/key.txt"); + + let op = S3Operation::ListBucket { + bucket: "b".into(), + raw_query: Some("prefix=foo/".into()), + }; + assert_eq!(op.key(), ""); + + assert_eq!(S3Operation::ListBuckets.key(), ""); + } +} From e314d1e54daa73292f0ab7b54488b423933740a4 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Feb 2026 22:43:56 -0800 Subject: [PATCH 05/82] More simplification --- crates/libs/core/src/proxy.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 9256aa1..18fd13e 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -664,21 +664,12 @@ fn build_backend_url( let bucket = &config.backend_bucket; let bucket_is_empty = bucket.is_empty(); - let key = match operation { - S3Operation::CreateMultipartUpload { key, .. } - | S3Operation::UploadPart { key, .. } - | S3Operation::CompleteMultipartUpload { key, .. } - | S3Operation::AbortMultipartUpload { key, .. } => { - let mut full_key = String::new(); - if let Some(prefix) = &config.backend_prefix { - full_key.push_str(prefix.trim_end_matches('/')); - full_key.push('/'); - } - full_key.push_str(key); - full_key - } - _ => return Err(ProxyError::Internal("unexpected operation for multipart URL".into())), - }; + let mut key = String::new(); + if let Some(prefix) = &config.backend_prefix { + key.push_str(prefix.trim_end_matches('/')); + key.push('/'); + } + key.push_str(operation.key()); let mut url = if bucket_is_empty { format!("{}/{}", base, key) From a8cb28ed5a11044a072c01e7e8195066becb6ace Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 20 Feb 2026 22:47:33 -0800 Subject: [PATCH 06/82] Prefer structs over strings for xml generation --- crates/libs/core/src/proxy.rs | 86 ++++++++++------------ crates/libs/core/src/s3/response.rs | 109 ++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 46 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 18fd13e..3f3aa68 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -8,7 +8,9 @@ use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLO use crate::error::ProxyError; use crate::resolver::{ListRewrite, ResolvedAction, RequestResolver}; use crate::response_body::ProxyResponseBody; -use crate::s3::response::ErrorResponse; +use crate::s3::response::{ + ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents, +}; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; @@ -555,52 +557,44 @@ fn build_list_xml( format!("{}/", backend_prefix) }; - let mut xml = format!( - "\ - \ - {}\ - {}\ - {}\ - 1000\ - false\ - {}", - bucket_name, - client_prefix, - delimiter, - list_result.objects.len() + list_result.common_prefixes.len() - ); - - for obj in &list_result.objects { - let raw_key = obj.location.to_string(); - let key = rewrite_key(&raw_key, &strip_prefix, list_rewrite); - - let last_modified = obj.last_modified.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let etag = obj.e_tag.as_deref().unwrap_or("\"\""); - - xml.push_str(&format!( - "\ - {}\ - {}\ - {}\ - {}\ - STANDARD\ - ", - key, last_modified, etag, obj.size - )); - } - - for prefix_path in &list_result.common_prefixes { - let raw_prefix = format!("{}/", prefix_path); - let prefix = rewrite_key(&raw_prefix, &strip_prefix, list_rewrite); - - xml.push_str(&format!( - "{}", - prefix - )); + let contents: Vec = list_result + .objects + .iter() + .map(|obj| { + let raw_key = obj.location.to_string(); + ListContents { + key: rewrite_key(&raw_key, &strip_prefix, list_rewrite), + last_modified: obj.last_modified.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(), + etag: obj.e_tag.as_deref().unwrap_or("\"\"").to_string(), + size: obj.size, + storage_class: "STANDARD", + } + }) + .collect(); + + let common_prefixes: Vec = list_result + .common_prefixes + .iter() + .map(|p| { + let raw_prefix = format!("{}/", p); + ListCommonPrefix { + prefix: rewrite_key(&raw_prefix, &strip_prefix, list_rewrite), + } + }) + .collect(); + + ListBucketResult { + xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", + name: bucket_name.to_string(), + prefix: client_prefix.to_string(), + delimiter: delimiter.to_string(), + max_keys: 1000, + is_truncated: false, + key_count: contents.len() + common_prefixes.len(), + contents, + common_prefixes, } - - xml.push_str(""); - xml + .to_xml() } /// Apply strip/add prefix rewriting to a key or prefix value. diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs index f73391b..0e91db8 100644 --- a/crates/libs/core/src/s3/response.rs +++ b/crates/libs/core/src/s3/response.rs @@ -138,6 +138,59 @@ impl ListAllMyBucketsResult { } } +/// S3 ListObjectsV2 response. +#[derive(Debug, Serialize)] +#[serde(rename = "ListBucketResult")] +pub struct ListBucketResult { + #[serde(rename = "@xmlns")] + pub xmlns: &'static str, + #[serde(rename = "Name")] + pub name: String, + #[serde(rename = "Prefix")] + pub prefix: String, + #[serde(rename = "Delimiter")] + pub delimiter: String, + #[serde(rename = "MaxKeys")] + pub max_keys: usize, + #[serde(rename = "IsTruncated")] + pub is_truncated: bool, + #[serde(rename = "KeyCount")] + pub key_count: usize, + #[serde(rename = "Contents", default)] + pub contents: Vec, + #[serde(rename = "CommonPrefixes", default)] + pub common_prefixes: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ListContents { + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "LastModified")] + pub last_modified: String, + #[serde(rename = "ETag")] + pub etag: String, + #[serde(rename = "Size")] + pub size: u64, + #[serde(rename = "StorageClass")] + pub storage_class: &'static str, +} + +#[derive(Debug, Serialize)] +pub struct ListCommonPrefix { + #[serde(rename = "Prefix")] + pub prefix: String, +} + +impl ListBucketResult { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} + /// STS AssumeRoleWithWebIdentity response. #[derive(Debug, Serialize)] #[serde(rename = "AssumeRoleWithWebIdentityResponse")] @@ -182,3 +235,59 @@ impl AssumeRoleWithWebIdentityResponse { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_list_bucket_result_xml() { + let result = ListBucketResult { + xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", + name: "my-bucket".to_string(), + prefix: "photos/".to_string(), + delimiter: "/".to_string(), + max_keys: 1000, + is_truncated: false, + key_count: 1, + contents: vec![ListContents { + key: "photos/image.jpg".to_string(), + last_modified: "2024-01-01T00:00:00.000Z".to_string(), + etag: "\"abc123\"".to_string(), + size: 1024, + storage_class: "STANDARD", + }], + common_prefixes: vec![ListCommonPrefix { + prefix: "photos/thumbs/".to_string(), + }], + }; + + let xml = result.to_xml(); + assert!(xml.starts_with("")); + assert!(xml.contains("")); + assert!(xml.contains("my-bucket")); + assert!(xml.contains("photos/image.jpg")); + assert!(xml.contains("1024")); + assert!(xml.contains("photos/thumbs/")); + } + + #[test] + fn test_list_bucket_result_empty() { + let result = ListBucketResult { + xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", + name: "bucket".to_string(), + prefix: String::new(), + delimiter: "/".to_string(), + max_keys: 1000, + is_truncated: false, + key_count: 0, + contents: vec![], + common_prefixes: vec![], + }; + + let xml = result.to_xml(); + assert!(xml.contains("0")); + assert!(!xml.contains("")); + assert!(!xml.contains("")); + } +} From 9bee2aaa038e8f7f8a345220f6db3584278223e5 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 08:17:19 -0800 Subject: [PATCH 07/82] Describe reuse intentions --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 1028c93..ea07c7d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,8 @@ Multi-runtime S3 gateway proxy in Rust. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. Uses the `object_store` crate for high-level operations (GET, HEAD, PUT, LIST) and raw signed HTTP for multipart uploads. +The intention of this codebase is to serve as a data proxy for the Source Cooperative. However, it should be structured in a way for others to use and build upon for their individual proxy needs. As such, a modular approach should be utilized to enable others to compose similar but different sytems. + ## Workspace Structure - `crates/libs/core` — Core proxy logic, traits, config, S3 request parsing From db8475e249a853b3a7c3eb7cb661762d307a2560 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 10:36:39 -0800 Subject: [PATCH 08/82] Mv source backend to be a core module. Rework backend tooling to better support non-s3 backends --- CLAUDE.md | 9 +- Cargo.lock | 26 +++ Cargo.toml | 3 + README.md | 4 +- config.example.toml | 56 ++++-- crates/libs/core/Cargo.toml | 2 + crates/libs/core/src/backend.rs | 103 ++++++++++- crates/libs/core/src/config/static_file.rs | 13 +- crates/libs/core/src/proxy.rs | 25 ++- crates/libs/core/src/resolver.rs | 2 +- crates/libs/core/src/types.rs | 53 ++++-- crates/libs/source-coop/Cargo.toml | 15 ++ .../source-coop/src/api.rs} | 52 ++++-- crates/libs/source-coop/src/lib.rs | 2 + .../source-coop/src/resolver.rs} | 79 +++++++-- crates/runtimes/cf-workers/Cargo.toml | 3 +- crates/runtimes/cf-workers/src/client.rs | 164 ++++++++---------- crates/runtimes/cf-workers/src/lib.rs | 11 +- crates/runtimes/cf-workers/wrangler.toml | 50 +++--- crates/runtimes/server/Cargo.toml | 3 +- crates/runtimes/server/src/client.rs | 22 +-- 21 files changed, 476 insertions(+), 221 deletions(-) create mode 100644 crates/libs/source-coop/Cargo.toml rename crates/{runtimes/cf-workers/src/source_api.rs => libs/source-coop/src/api.rs} (80%) create mode 100644 crates/libs/source-coop/src/lib.rs rename crates/{runtimes/cf-workers/src/source_resolver.rs => libs/source-coop/src/resolver.rs} (81%) diff --git a/CLAUDE.md b/CLAUDE.md index ea07c7d..092aa1e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,8 +28,9 @@ cargo test ## Key Architecture Notes - **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has two methods: `create_store()` returns an `Arc` for high-level operations, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: - - **Server**: `ServerBackend` uses the default `object_store` HTTP connector (reqwest-based) and reqwest for raw HTTP. - - **CF Workers**: `WorkerBackend` uses a custom `FetchConnector` (implementing `object_store::client::HttpConnector`) that bridges the Workers Fetch API to `object_store`, and `web_sys::fetch` for raw HTTP. + - **Server**: `ServerBackend` delegates to `build_object_store()` with identity connector and uses reqwest for raw HTTP. + - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` injecting `FetchConnector`, and uses `web_sys::fetch` for raw HTTP. +- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. The `build_object_store()` function in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterates `backend_options` calling `with_config()` on the appropriate builder (`AmazonS3Builder`, `MicrosoftAzureBuilder`, `GoogleCloudStorageBuilder`). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder`. - **Operation dispatch**: The proxy handler dispatches S3 operations to different backends: - **GET** → `store.get_opts()` with Range/If-Match/If-None-Match header parsing; returns `ProxyResponseBody::Stream`. - **HEAD** → `store.head()`; returns metadata headers + empty body. @@ -47,6 +48,6 @@ cargo test ## Known Limitations -1. **Multipart uses raw HTTP**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP, which is S3-specific. +1. **Multipart uses raw HTTP (S3 only)**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP. They are gated to `backend_type == "s3"` — non-S3 backends return an error for multipart requests and should use `PUT` (object_store handles chunking internally). 2. **LIST returns all results**: `object_store::list_with_delimiter()` fetches all pages internally. No S3-style pagination (continuation tokens, max-keys truncation). `IsTruncated` is always `false`. -3. **S3 only**: `BucketConfig` and store creation assume S3. Adding GCS/Azure requires a `backend_type` field and builder dispatch. +3. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. diff --git a/Cargo.lock b/Cargo.lock index 1187832..490910f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1517,6 +1517,7 @@ dependencies = [ "futures", "http 1.4.0", "http-body-util", + "httparse", "humantime", "hyper 1.8.1", "itertools", @@ -1527,6 +1528,7 @@ dependencies = [ "rand 0.9.2", "reqwest", "ring", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -2023,6 +2025,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -2104,6 +2115,7 @@ dependencies = [ "quick-xml 0.37.5", "s3-proxy-auth", "s3-proxy-core", + "s3-proxy-source-coop", "serde", "serde_json", "thiserror", @@ -2157,6 +2169,7 @@ dependencies = [ "reqwest", "s3-proxy-auth", "s3-proxy-core", + "s3-proxy-source-coop", "serde", "thiserror", "tokio", @@ -2165,6 +2178,19 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "s3-proxy-source-coop" +version = "0.1.0" +dependencies = [ + "bytes", + "http 1.4.0", + "s3-proxy-core", + "serde", + "serde_json", + "tracing", + "url", +] + [[package]] name = "schannel" version = "0.1.28" diff --git a/Cargo.toml b/Cargo.toml index 5308b6b..04c89dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/libs/core", "crates/libs/auth", + "crates/libs/source-coop", "crates/runtimes/server", "crates/runtimes/cf-workers", ] @@ -11,6 +12,7 @@ members = [ default-members = [ "crates/libs/core", "crates/libs/auth", + "crates/libs/source-coop", "crates/runtimes/server", ] resolver = "2" @@ -69,3 +71,4 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Internal crates s3-proxy-core = { path = "crates/libs/core" } s3-proxy-auth = { path = "crates/libs/auth" } +s3-proxy-source-coop = { path = "crates/libs/source-coop" } diff --git a/README.md b/README.md index bdd3c9f..0ea6f6e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A multi-runtime S3 gateway that streams requests to and from backing object stor ## Architecture ```mermaid -flowchart LR +flowchart Clients["S3 Clients
(aws cli, boto3, sdk, etc.)"] subgraph Proxy["s3-proxy-rs"] @@ -22,7 +22,7 @@ flowchart LR ### Crate Layout -``` +```sh crates/ ├── libs/ # Libraries — not directly runnable │ ├── core/ (s3-proxy-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config providers diff --git a/config.example.toml b/config.example.toml index 975222c..4ffabcb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -7,40 +7,60 @@ # Virtual Buckets # ============================================================================= -# A publicly accessible bucket (anonymous reads allowed) +# A publicly accessible S3 bucket (anonymous reads allowed) [[buckets]] name = "public-data" -backend_endpoint = "https://s3.us-east-1.amazonaws.com" -backend_bucket = "my-company-public-assets" -backend_region = "us-east-1" -backend_access_key_id = "AKIAIOSFODNN7EXAMPLE" -backend_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +backend_type = "s3" anonymous_access = true allowed_roles = [] -# A private bucket backed by MinIO +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-company-public-assets" +region = "us-east-1" +access_key_id = "AKIAIOSFODNN7EXAMPLE" +secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +# A private S3 bucket backed by MinIO [[buckets]] name = "ml-artifacts" -backend_endpoint = "https://minio.internal:9000" -backend_bucket = "ml-pipeline-artifacts" +backend_type = "s3" backend_prefix = "v2" -backend_region = "us-east-1" -backend_access_key_id = "minioadmin" -backend_secret_access_key = "minioadmin" anonymous_access = false allowed_roles = ["github-actions-deployer"] -# Another private bucket on a different S3-compatible service +[buckets.backend_options] +endpoint = "https://minio.internal:9000" +bucket_name = "ml-pipeline-artifacts" +region = "us-east-1" +access_key_id = "minioadmin" +secret_access_key = "minioadmin" + +# An S3 bucket on a different region [[buckets]] name = "deploy-bundles" -backend_endpoint = "https://s3.us-west-2.amazonaws.com" -backend_bucket = "prod-deploy-bundles" -backend_region = "us-west-2" -backend_access_key_id = "AKIAI44QH8DHBEXAMPLE" -backend_secret_access_key = "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" +backend_type = "s3" anonymous_access = false allowed_roles = ["github-actions-deployer", "ci-readonly"] +[buckets.backend_options] +endpoint = "https://s3.us-west-2.amazonaws.com" +bucket_name = "prod-deploy-bundles" +region = "us-west-2" +access_key_id = "AKIAI44QH8DHBEXAMPLE" +secret_access_key = "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + +# An Azure Blob Storage backend +[[buckets]] +name = "azure-data" +backend_type = "az" +anonymous_access = true +allowed_roles = [] + +[buckets.backend_options] +account_name = "mystorageaccount" +container_name = "public-datasets" + # ============================================================================= # IAM Roles (for STS AssumeRoleWithWebIdentity) # ============================================================================= diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml index 58799bc..2dec1a8 100644 --- a/crates/libs/core/Cargo.toml +++ b/crates/libs/core/Cargo.toml @@ -10,6 +10,8 @@ default = [] config-dynamodb = ["aws-sdk-dynamodb", "tokio"] config-postgres = ["sqlx"] config-http = ["reqwest"] +azure = ["object_store/azure"] +gcp = ["object_store/gcp"] [dependencies] async-trait.workspace = true diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index 966080b..bd8faed 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -9,16 +9,23 @@ //! covered by `ObjectStore` (multipart uploads). //! //! [`S3RequestSigner`] is retained for signing multipart requests. +//! [`build_object_store`] dispatches on `BucketConfig::backend_type` to build +//! the appropriate provider store. use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; +use crate::types::{BackendType, BucketConfig}; use bytes::Bytes; use http::HeaderMap; +use object_store::aws::AmazonS3Builder; use object_store::ObjectStore; use std::future::Future; use std::sync::Arc; -use crate::types::BucketConfig; +#[cfg(feature = "azure")] +use object_store::azure::MicrosoftAzureBuilder; +#[cfg(feature = "gcp")] +use object_store::gcp::GoogleCloudStorageBuilder; /// Trait for runtime-specific backend operations. /// @@ -47,6 +54,100 @@ pub struct RawResponse { pub body: Bytes, } +/// Wrapper around provider-specific `object_store` builders. +/// +/// Runtimes use [`build_object_store`] and inject their HTTP connector via +/// a closure that receives this enum. +pub enum StoreBuilder { + S3(AmazonS3Builder), + #[cfg(feature = "azure")] + Azure(MicrosoftAzureBuilder), + #[cfg(feature = "gcp")] + Gcs(GoogleCloudStorageBuilder), +} + +impl StoreBuilder { + /// Build the final `ObjectStore`. + pub fn build(self) -> Result, ProxyError> { + match self { + StoreBuilder::S3(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, + )), + #[cfg(feature = "azure")] + StoreBuilder::Azure(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build Azure store: {}", e)))?, + )), + #[cfg(feature = "gcp")] + StoreBuilder::Gcs(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build GCS store: {}", e)))?, + )), + } + } +} + +/// Build an `ObjectStore` from a [`BucketConfig`], dispatching on `backend_type`. +/// +/// The `configure` closure lets each runtime inject its HTTP connector: +/// - Server runtime passes `|b| b` (default connector) +/// - CF Workers passes `|b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), .. }` +pub fn build_object_store(config: &BucketConfig, configure: F) -> Result, ProxyError> +where + F: FnOnce(StoreBuilder) -> StoreBuilder, +{ + let backend_type = config + .parsed_backend_type() + .ok_or_else(|| ProxyError::ConfigError(format!("unsupported backend_type: '{}'", config.backend_type)))?; + + let builder = match backend_type { + BackendType::S3 => { + let mut b = AmazonS3Builder::new(); + for (k, v) in &config.backend_options { + if let Ok(key) = k.parse() { + b = b.with_config(key, v); + } + } + StoreBuilder::S3(b) + } + #[cfg(feature = "azure")] + BackendType::Azure => { + let mut b = MicrosoftAzureBuilder::new(); + for (k, v) in &config.backend_options { + if let Ok(key) = k.parse() { + b = b.with_config(key, v); + } + } + StoreBuilder::Azure(b) + } + #[cfg(not(feature = "azure"))] + BackendType::Azure => { + return Err(ProxyError::ConfigError( + "Azure backend support not enabled (requires 'azure' feature)".into(), + )); + } + #[cfg(feature = "gcp")] + BackendType::Gcs => { + let mut b = GoogleCloudStorageBuilder::new(); + for (k, v) in &config.backend_options { + if let Ok(key) = k.parse() { + b = b.with_config(key, v); + } + } + StoreBuilder::Gcs(b) + } + #[cfg(not(feature = "gcp"))] + BackendType::Gcs => { + return Err(ProxyError::ConfigError( + "GCS backend support not enabled (requires 'gcp' feature)".into(), + )); + } + }; + + configure(builder).build() +} + /// Helper to build a signed URL + headers for an outbound request to S3. /// /// Used for multipart operations (CreateMultipartUpload, UploadPart, diff --git a/crates/libs/core/src/config/static_file.rs b/crates/libs/core/src/config/static_file.rs index 8f6e35f..63b9b57 100644 --- a/crates/libs/core/src/config/static_file.rs +++ b/crates/libs/core/src/config/static_file.rs @@ -29,13 +29,16 @@ pub struct StaticConfig { /// let provider = StaticProvider::from_toml(r#" /// [[buckets]] /// name = "public-data" -/// backend_endpoint = "https://s3.amazonaws.com" -/// backend_bucket = "my-real-bucket" -/// backend_region = "us-east-1" -/// backend_access_key_id = "AKIA..." -/// backend_secret_access_key = "..." +/// backend_type = "s3" /// anonymous_access = true /// allowed_roles = [] +/// +/// [buckets.backend_options] +/// endpoint = "https://s3.amazonaws.com" +/// bucket_name = "my-real-bucket" +/// region = "us-east-1" +/// access_key_id = "AKIA..." +/// secret_access_key = "..." /// "#)?; /// ``` #[derive(Clone)] diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 3f3aa68..6e4da8d 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -150,11 +150,17 @@ where self.handle_list(bucket_config, raw_query.as_deref(), list_rewrite) .await } - // Multipart operations go through raw signed HTTP + // Multipart operations go through raw signed HTTP (S3 only) S3Operation::CreateMultipartUpload { .. } | S3Operation::UploadPart { .. } | S3Operation::CompleteMultipartUpload { .. } | S3Operation::AbortMultipartUpload { .. } => { + if !bucket_config.supports_s3_multipart() { + return Err(ProxyError::InvalidRequest(format!( + "multipart operations not supported for '{}' backends", + bucket_config.backend_type + ))); + } self.handle_multipart(method, operation, bucket_config, original_headers, body) .await } @@ -443,8 +449,10 @@ where } // Sign the request if credentials are configured - let has_credentials = !bucket_config.backend_access_key_id.is_empty() - && !bucket_config.backend_secret_access_key.is_empty(); + let access_key = bucket_config.option("access_key_id").unwrap_or(""); + let secret_key = bucket_config.option("secret_access_key").unwrap_or(""); + let region = bucket_config.option("region").unwrap_or("us-east-1"); + let has_credentials = !access_key.is_empty() && !secret_key.is_empty(); let parsed_url = Url::parse(&backend_url) .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; @@ -457,9 +465,9 @@ where if has_credentials { let signer = S3RequestSigner::new( - bucket_config.backend_access_key_id.clone(), - bucket_config.backend_secret_access_key.clone(), - bucket_config.backend_region.clone(), + access_key.to_string(), + secret_key.to_string(), + region.to_string(), ); signer.sign_request(method, &parsed_url, &mut headers, &payload_hash)?; } else { @@ -654,8 +662,9 @@ fn build_backend_url( config: &BucketConfig, operation: &S3Operation, ) -> Result { - let base = config.backend_endpoint.trim_end_matches('/'); - let bucket = &config.backend_bucket; + let endpoint = config.option("endpoint").unwrap_or(""); + let base = endpoint.trim_end_matches('/'); + let bucket = config.option("bucket_name").unwrap_or(""); let bucket_is_empty = bucket.is_empty(); let mut key = String::new(); diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index 834ae08..06d162d 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -144,7 +144,7 @@ impl RequestResolver for DefaultResolver

{ tracing::debug!( bucket = %bucket_name, - backend_endpoint = %bucket_config.backend_endpoint, + backend_type = %bucket_config.backend_type, "resolved bucket config" ); diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index fd804da..db4827e 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -2,6 +2,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; /// Configuration for a virtual bucket exposed by the proxy. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -9,28 +10,56 @@ pub struct BucketConfig { /// The virtual bucket name exposed to clients. pub name: String, - /// The backing object store endpoint (e.g., "https://s3.amazonaws.com"). - pub backend_endpoint: String, - - /// The real bucket name on the backing store. - pub backend_bucket: String, + /// Provider type: "s3", "az", "gcs", etc. + pub backend_type: String, /// Optional prefix to prepend to all keys when forwarding. pub backend_prefix: Option, - /// The region to use when signing requests to the backend. - pub backend_region: String, - - /// Credentials for signing outbound requests to the backing store. - pub backend_access_key_id: String, - pub backend_secret_access_key: String, - /// Whether this bucket allows anonymous (unsigned) access. pub anonymous_access: bool, /// IAM role ARNs that are allowed to access this bucket. /// Empty means only anonymous access (if enabled) or long-lived credentials. pub allowed_roles: Vec, + + /// Provider-specific config passed to the object_store builder. + /// Keys are the short aliases accepted by each provider's ConfigKey::from_str(). + /// S3: "endpoint", "bucket_name", "region", "access_key_id", "secret_access_key", "skip_signature" + /// Azure: "account_name", "container_name", "access_key", "skip_signature" + /// GCS: "bucket_name", "service_account_key", "skip_signature" + #[serde(default)] + pub backend_options: HashMap, +} + +/// Known backend provider types. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendType { + S3, + Azure, + Gcs, +} + +impl BucketConfig { + /// Parse the `backend_type` string into a known [`BackendType`]. + pub fn parsed_backend_type(&self) -> Option { + match self.backend_type.as_str() { + "s3" => Some(BackendType::S3), + "az" | "azure" => Some(BackendType::Azure), + "gcs" | "gs" => Some(BackendType::Gcs), + _ => None, + } + } + + /// Whether this backend supports S3-style multipart uploads via raw HTTP. + pub fn supports_s3_multipart(&self) -> bool { + matches!(self.parsed_backend_type(), Some(BackendType::S3)) + } + + /// Look up a value in `backend_options`. + pub fn option(&self, key: &str) -> Option<&str> { + self.backend_options.get(key).map(|s| s.as_str()) + } } /// Configuration for an IAM role that can be assumed via STS. diff --git a/crates/libs/source-coop/Cargo.toml b/crates/libs/source-coop/Cargo.toml new file mode 100644 index 0000000..50050d0 --- /dev/null +++ b/crates/libs/source-coop/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "s3-proxy-source-coop" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Source Cooperative API client and request resolver for the S3 proxy gateway" + +[dependencies] +s3-proxy-core.workspace = true +serde.workspace = true +serde_json.workspace = true +http.workspace = true +bytes.workspace = true +url.workspace = true +tracing.workspace = true diff --git a/crates/runtimes/cf-workers/src/source_api.rs b/crates/libs/source-coop/src/api.rs similarity index 80% rename from crates/runtimes/cf-workers/src/source_api.rs rename to crates/libs/source-coop/src/api.rs index 8ddc5e4..326df35 100644 --- a/crates/runtimes/cf-workers/src/source_api.rs +++ b/crates/libs/source-coop/src/api.rs @@ -1,13 +1,31 @@ //! HTTP client for the Source Cooperative API. //! //! Makes server-to-server calls to resolve products, data connections, -//! API keys, and permissions. All calls use the Workers Fetch API via -//! the shared `fetch_json` helper in `client.rs`. +//! API keys, and permissions. The actual HTTP transport is abstracted behind +//! the [`HttpClient`] trait so each runtime can provide its own implementation. -use crate::client::{fetch_json, CacheOptions}; use s3_proxy_core::error::ProxyError; +use s3_proxy_core::maybe_send::{MaybeSend, MaybeSync}; +use serde::de::DeserializeOwned; use serde::Deserialize; use std::collections::HashMap; +use std::future::Future; + +/// Options for response caching. +pub struct CacheOptions { + pub cache_ttl: u32, + pub cache_key: Option, +} + +/// Trait abstracting HTTP JSON fetching so each runtime can provide its own implementation. +pub trait HttpClient: Clone + MaybeSend + MaybeSync + 'static { + fn fetch_json( + &self, + url: &str, + headers: &[(&str, &str)], + cache: Option<&CacheOptions>, + ) -> impl Future> + MaybeSend; +} /// Per-endpoint cache TTLs (seconds). Set to 0 to disable caching. pub struct CacheTtls { @@ -32,7 +50,8 @@ impl Default for CacheTtls { /// Client for the Source Cooperative API. #[derive(Clone)] -pub struct SourceApiClient { +pub struct SourceApiClient { + http: H, api_url: String, api_key: String, product_cache_ttl: u32, @@ -42,7 +61,7 @@ pub struct SourceApiClient { api_key_cache_ttl: u32, } -// ── API response types ────────────────────────────────────────────── +// -- API response types -- #[derive(Debug, Deserialize)] pub struct SourceProduct { @@ -76,6 +95,10 @@ pub struct ConnectionDetails { pub region: Option, pub base_prefix: Option, pub bucket: Option, + #[serde(default)] + pub account_name: Option, + #[serde(default)] + pub container_name: Option, } #[derive(Debug, Deserialize)] @@ -84,6 +107,8 @@ pub struct ConnectionAuth { pub auth_type: String, pub access_key_id: Option, pub secret_access_key: Option, + #[serde(default)] + pub access_key: Option, } #[derive(Debug, Deserialize)] @@ -120,11 +145,12 @@ pub struct AccountRepository { pub repository_id: String, } -// ── Client implementation ─────────────────────────────────────────── +// -- Client implementation -- -impl SourceApiClient { - pub fn new(api_url: String, api_key: String, cache_ttls: CacheTtls) -> Self { +impl SourceApiClient { + pub fn new(http: H, api_url: String, api_key: String, cache_ttls: CacheTtls) -> Self { Self { + http, api_url, api_key, product_cache_ttl: cache_ttls.product, @@ -155,7 +181,7 @@ impl SourceApiClient { cache_ttl: self.product_cache_ttl, cache_key: None, }); - fetch_json(&url, &headers, cache.as_ref()).await + self.http.fetch_json(&url, &headers, cache.as_ref()).await } /// `GET /api/v1/data-connections/{id}` @@ -167,7 +193,7 @@ impl SourceApiClient { cache_ttl: self.data_connection_cache_ttl, cache_key: None, }); - fetch_json(&url, &headers, cache.as_ref()).await + self.http.fetch_json(&url, &headers, cache.as_ref()).await } /// `GET /api/v1/api-keys/{access_key_id}/auth` @@ -179,7 +205,7 @@ impl SourceApiClient { cache_ttl: self.api_key_cache_ttl, cache_key: None, }); - fetch_json(&url, &headers, cache.as_ref()).await + self.http.fetch_json(&url, &headers, cache.as_ref()).await } /// `GET /api/v1/products/{account_id}/{repo_id}/permissions` @@ -207,7 +233,7 @@ impl SourceApiClient { account_id, repo_id, user_api_key )), }); - fetch_json(&url, &headers, cache.as_ref()).await + self.http.fetch_json(&url, &headers, cache.as_ref()).await } /// `GET /api/v1/accounts/{account_id}` @@ -222,6 +248,6 @@ impl SourceApiClient { cache_ttl: self.account_cache_ttl, cache_key: None, }); - fetch_json(&url, &headers, cache.as_ref()).await + self.http.fetch_json(&url, &headers, cache.as_ref()).await } } diff --git a/crates/libs/source-coop/src/lib.rs b/crates/libs/source-coop/src/lib.rs new file mode 100644 index 0000000..57563d8 --- /dev/null +++ b/crates/libs/source-coop/src/lib.rs @@ -0,0 +1,2 @@ +pub mod api; +pub mod resolver; diff --git a/crates/runtimes/cf-workers/src/source_resolver.rs b/crates/libs/source-coop/src/resolver.rs similarity index 81% rename from crates/runtimes/cf-workers/src/source_resolver.rs rename to crates/libs/source-coop/src/resolver.rs index 73f0e28..f56fda5 100644 --- a/crates/runtimes/cf-workers/src/source_resolver.rs +++ b/crates/libs/source-coop/src/resolver.rs @@ -2,15 +2,16 @@ //! //! Consolidates all Source Cooperative business logic (URL namespace mapping, //! external API auth, query/response prefix rewriting, synthetic XML responses) -//! into a single resolver that the thin CF Workers adapter calls. +//! into a single resolver that thin runtime adapters call. -use crate::source_api::SourceApiClient; +use crate::api::{HttpClient, SourceApiClient}; use bytes::Bytes; use http::{HeaderMap, Method}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use s3_proxy_core::s3::request::build_s3_operation; use s3_proxy_core::types::BucketConfig; +use std::collections::HashMap; /// Request resolver for Source Cooperative. /// @@ -19,12 +20,12 @@ use s3_proxy_core::types::BucketConfig; /// - `/{account_id}` -> synthetic account listing or list-with-prefix /// - `/{account_id}/{repo_id}[/{key}]` -> proxy to backend #[derive(Clone)] -pub struct SourceCoopResolver { - api_client: SourceApiClient, +pub struct SourceCoopResolver { + api_client: SourceApiClient, } -impl SourceCoopResolver { - pub fn new(api_client: SourceApiClient) -> Self { +impl SourceCoopResolver { + pub fn new(api_client: SourceApiClient) -> Self { Self { api_client } } @@ -70,8 +71,6 @@ impl SourceCoopResolver { .get_data_connection(&mirror.connection_id) .await?; - let region = conn.details.region.unwrap_or_else(|| "us-east-1".into()); - let bucket = conn.details.bucket.unwrap_or_default(); let base_prefix = conn.details.base_prefix.unwrap_or_default(); let backend_prefix = { @@ -88,18 +87,63 @@ impl SourceCoopResolver { } }; - let endpoint = format!("https://s3.{}.amazonaws.com", region); + let provider = conn.details.provider.as_str(); + let backend_options = match provider { + "s3" => { + let region = conn.details.region.unwrap_or_else(|| "us-east-1".into()); + let bucket = conn.details.bucket.unwrap_or_default(); + let endpoint = format!("https://s3.{}.amazonaws.com", region); + let mut opts = HashMap::new(); + opts.insert("endpoint".to_string(), endpoint); + opts.insert("bucket_name".to_string(), bucket); + opts.insert("region".to_string(), region); + if let Some(ak) = &conn.authentication.access_key_id { + opts.insert("access_key_id".to_string(), ak.clone()); + } + if let Some(sk) = &conn.authentication.secret_access_key { + opts.insert("secret_access_key".to_string(), sk.clone()); + } + if conn.authentication.access_key_id.is_none() { + opts.insert("skip_signature".to_string(), "true".to_string()); + } + opts + } + "az" | "azure" => { + let mut opts = HashMap::new(); + if let Some(name) = &conn.details.account_name { + opts.insert("account_name".to_string(), name.clone()); + } + if let Some(container) = &conn.details.container_name { + opts.insert("container_name".to_string(), container.clone()); + } + if let Some(key) = &conn.authentication.access_key { + opts.insert("access_key".to_string(), key.clone()); + } + if conn.authentication.access_key.is_none() { + opts.insert("skip_signature".to_string(), "true".to_string()); + } + opts + } + other => { + return Err(ProxyError::ConfigError(format!( + "unsupported provider '{}' for data connection", + other + ))); + } + }; + + let backend_type = match provider { + "az" | "azure" => "az".to_string(), + other => other.to_string(), + }; Ok(BucketConfig { name: bucket_name, - backend_endpoint: endpoint, - backend_bucket: bucket, + backend_type, backend_prefix, - backend_region: region, - backend_access_key_id: conn.authentication.access_key_id.unwrap_or_default(), - backend_secret_access_key: conn.authentication.secret_access_key.unwrap_or_default(), anonymous_access: product.data_mode == "open", allowed_roles: vec![], + backend_options, }) } @@ -213,7 +257,7 @@ impl SourceCoopResolver { } } -impl RequestResolver for SourceCoopResolver { +impl RequestResolver for SourceCoopResolver { async fn resolve( &self, method: &Method, @@ -302,13 +346,10 @@ impl RequestResolver for SourceCoopResolver { } } -// ── Helpers ───────────────────────────────────────────────────────── +// -- Helpers -- /// Build a [`ListRewrite`] that strips the backend prefix and prepends `repo_id`. fn build_list_rewrite(bucket_config: &BucketConfig, repo_id: &str) -> Option { - // The backend prefix is what core prepends to list queries. We need to - // strip it from response keys and add `repo_id/` so clients see the - // expected namespace. let strip = bucket_config .backend_prefix .as_deref() diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index 829a75e..b945285 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] [dependencies] s3-proxy-core = { workspace = true, features = [] } s3-proxy-auth.workspace = true +s3-proxy-source-coop.workspace = true bytes.workspace = true http.workspace = true serde.workspace = true @@ -20,7 +21,7 @@ thiserror.workspace = true chrono.workspace = true quick-xml.workspace = true url.workspace = true -object_store.workspace = true +object_store = { version = "0.12", default-features = false, features = ["aws"] } futures.workspace = true http-body.workspace = true http-body-util.workspace = true diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 9e76714..dc0b834 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -2,26 +2,20 @@ //! //! Contains: //! - `WorkerBackend` — implements `ProxyBackend` using the Fetch API + FetchConnector -//! - `fetch_json` — helper for server-to-server API calls (used by `source_api`) +//! - `WorkerHttpClient` — implements `HttpClient` for server-to-server API calls use crate::fetch_connector::FetchConnector; use bytes::Bytes; use http::HeaderMap; -use object_store::aws::AmazonS3Builder; use object_store::ObjectStore; -use s3_proxy_core::backend::{ProxyBackend, RawResponse}; +use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse, StoreBuilder}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; +use s3_proxy_source_coop::api::{CacheOptions, HttpClient}; use serde::de::DeserializeOwned; use std::sync::Arc; use worker::{Cache, Fetch}; -/// Options for Cache API caching. -pub(crate) struct CacheOptions { - pub cache_ttl: u32, - pub cache_key: Option, -} - /// Build the cache key URL for the Cache API. /// /// When a custom key is provided, it is formatted as `https://cache.internal/{key}` @@ -33,82 +27,86 @@ fn cache_key_url(url: &str, opts: &CacheOptions) -> String { } } -/// Fetch a URL and deserialize the JSON response body. +/// HTTP client for the Cloudflare Workers runtime. /// -/// Used by `source_api` for server-to-server calls to the Source Cooperative API. -/// When `cache` is provided, responses are cached using the Cloudflare Workers -/// Cache API with explicit `cache.get()` / `cache.put()` calls. -pub(crate) async fn fetch_json( - url: &str, - headers: &[(&str, &str)], - cache: Option<&CacheOptions>, -) -> Result { - // Check cache for a hit before making the request. - let cache_state = if let Some(opts) = cache { - let key = cache_key_url(url, opts); - let cf_cache = Cache::default(); - match cf_cache.get(&key, false).await { - Ok(Some(mut cached)) => { - if let Ok(text) = cached.text().await { - if let Ok(value) = serde_json::from_str(&text) { - return Ok(value); +/// Uses the Workers Fetch API for requests and the Cache API for caching. +#[derive(Clone)] +pub struct WorkerHttpClient; + +impl HttpClient for WorkerHttpClient { + async fn fetch_json( + &self, + url: &str, + headers: &[(&str, &str)], + cache: Option<&CacheOptions>, + ) -> Result { + // Check cache for a hit before making the request. + let cache_state = if let Some(opts) = cache { + let key = cache_key_url(url, opts); + let cf_cache = Cache::default(); + match cf_cache.get(&key, false).await { + Ok(Some(mut cached)) => { + if let Ok(text) = cached.text().await { + if let Ok(value) = serde_json::from_str(&text) { + return Ok(value); + } } + // Cache hit but couldn't deserialize — fall through to fetch. + Some((cf_cache, key)) } - // Cache hit but couldn't deserialize — fall through to fetch. - Some((cf_cache, key)) + _ => Some((cf_cache, key)), } - _ => Some((cf_cache, key)), + } else { + None + }; + + // Build and execute the fetch request. + let mut req_init = worker::RequestInit::new(); + let worker_headers = worker::Headers::new(); + for (k, v) in headers { + worker_headers + .set(k, v) + .map_err(|e| ProxyError::Internal(format!("failed to set header: {}", e)))?; } - } else { - None - }; - - // Build and execute the fetch request. - let mut req_init = worker::RequestInit::new(); - let worker_headers = worker::Headers::new(); - for (k, v) in headers { - worker_headers - .set(k, v) - .map_err(|e| ProxyError::Internal(format!("failed to set header: {}", e)))?; - } - req_init.with_headers(worker_headers); + req_init.with_headers(worker_headers); - let req = worker::Request::new_with_init(url, &req_init) - .map_err(|e| ProxyError::Internal(format!("failed to create request: {}", e)))?; + let req = worker::Request::new_with_init(url, &req_init) + .map_err(|e| ProxyError::Internal(format!("failed to create request: {}", e)))?; - let mut resp = Fetch::Request(req) - .send() - .await - .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; + let mut resp = Fetch::Request(req) + .send() + .await + .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; - let status = resp.status_code(); + let status = resp.status_code(); - let text = resp - .text() - .await - .map_err(|e| ProxyError::Internal(format!("failed to read text: {}", e)))?; + let text = resp + .text() + .await + .map_err(|e| ProxyError::Internal(format!("failed to read text: {}", e)))?; - if status < 200 || status >= 300 { - return Err(ProxyError::BackendError(format!( - "API request to {} returned status {}", - url, status - ))); - } + if status < 200 || status >= 300 { + return Err(ProxyError::BackendError(format!( + "API request to {} returned status {}", + url, status + ))); + } - // Cache successful responses via the Cache API. - if let Some((cf_cache, key)) = cache_state { - let ttl = cache.unwrap().cache_ttl; - if let Ok(mut response) = worker::Response::ok(&text) { - let _ = response - .headers_mut() - .set("Cache-Control", &format!("max-age={}", ttl)); - // cache.put is fire-and-forget; ignore errors. - let _ = cf_cache.put(&key, response).await; + // Cache successful responses via the Cache API. + if let Some((cf_cache, key)) = cache_state { + let ttl = cache.unwrap().cache_ttl; + if let Ok(mut response) = worker::Response::ok(&text) { + let _ = response + .headers_mut() + .set("Cache-Control", &format!("max-age={}", ttl)); + // cache.put is fire-and-forget; ignore errors. + let _ = cf_cache.put(&key, response).await; + } } - } - serde_json::from_str(&text) - .map_err(|e| ProxyError::Internal(format!("failed to deserialize response: {}", e))) + serde_json::from_str(&text) + .map_err(|e| ProxyError::Internal(format!("failed to deserialize response: {}", e))) + } } /// Backend for the Cloudflare Workers runtime. @@ -120,25 +118,9 @@ pub struct WorkerBackend; impl ProxyBackend for WorkerBackend { fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { - let mut builder = AmazonS3Builder::new() - .with_endpoint(&config.backend_endpoint) - .with_bucket_name(&config.backend_bucket) - .with_region(&config.backend_region) - .with_http_connector(FetchConnector); - - if !config.backend_access_key_id.is_empty() { - builder = builder - .with_access_key_id(&config.backend_access_key_id) - .with_secret_access_key(&config.backend_secret_access_key); - } else { - builder = builder.with_skip_signature(true); - } - - Ok(Arc::new( - builder - .build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, - )) + build_object_store(config, |b| match b { + StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), + }) } async fn send_raw( diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index b57321f..2c4202d 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -25,14 +25,14 @@ mod body; mod client; mod fetch_connector; -mod source_api; -mod source_resolver; mod tracing_layer; use body::build_worker_response; use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; use s3_proxy_core::proxy::ProxyHandler; use s3_proxy_core::resolver::DefaultResolver; +use s3_proxy_source_coop::api::{CacheTtls, SourceApiClient}; +use s3_proxy_source_coop::resolver::SourceCoopResolver; use worker::*; /// The Worker entry point. @@ -75,7 +75,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { )) })?; - let mut cache_ttls = source_api::CacheTtls::default(); + let mut cache_ttls = CacheTtls::default(); if let Ok(v) = env.var("SOURCE_CACHE_TTL_PRODUCT") { if let Ok(n) = v.to_string().parse::() { cache_ttls.product = n; @@ -102,9 +102,10 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { } } + let http_client = client::WorkerHttpClient; let api_client = - source_api::SourceApiClient::new(source_api_url.to_string(), source_api_key, cache_ttls); - let resolver = source_resolver::SourceCoopResolver::new(api_client); + SourceApiClient::new(http_client, source_api_url.to_string(), source_api_key, cache_ttls); + let resolver = SourceCoopResolver::new(api_client); let handler = ProxyHandler::new(client::WorkerBackend, resolver); let result = handler diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index 028b975..e327d0d 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -15,45 +15,55 @@ roles = [] [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true -backend_access_key_id = "minioadmin" -backend_bucket = "public-data" -backend_endpoint = "http://localhost:9000" -backend_region = "us-east-1" -backend_secret_access_key = "minioadmin" +backend_type = "s3" name = "public-data" +[vars.PROXY_CONFIG.buckets.backend_options] +access_key_id = "minioadmin" +bucket_name = "public-data" +endpoint = "http://localhost:9000" +region = "us-east-1" +secret_access_key = "minioadmin" + [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = false -backend_access_key_id = "minioadmin" -backend_bucket = "private-uploads" -backend_endpoint = "http://localhost:9000" -backend_region = "us-east-1" -backend_secret_access_key = "minioadmin" +backend_type = "s3" name = "private-uploads" +[vars.PROXY_CONFIG.buckets.backend_options] +access_key_id = "minioadmin" +bucket_name = "private-uploads" +endpoint = "http://localhost:9000" +region = "us-east-1" +secret_access_key = "minioadmin" + [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true -backend_access_key_id = "" -backend_bucket = "us-west-2.opendata.source.coop" -backend_endpoint = "https://s3.us-west-2.amazonaws.com" +backend_type = "s3" backend_prefix = "cholmes/" -backend_region = "us-west-2" -backend_secret_access_key = "" name = "cholmes" +[vars.PROXY_CONFIG.buckets.backend_options] +bucket_name = "us-west-2.opendata.source.coop" +endpoint = "https://s3.us-west-2.amazonaws.com" +region = "us-west-2" +skip_signature = "true" + [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true -backend_access_key_id = "" -backend_bucket = "us-west-2.opendata.source.coop" -backend_endpoint = "https://s3.us-west-2.amazonaws.com" +backend_type = "s3" backend_prefix = "harvard-lil/" -backend_region = "us-west-2" -backend_secret_access_key = "" name = "harvard-lil" +[vars.PROXY_CONFIG.buckets.backend_options] +bucket_name = "us-west-2.opendata.source.coop" +endpoint = "https://s3.us-west-2.amazonaws.com" +region = "us-west-2" +skip_signature = "true" + [[vars.PROXY_CONFIG.credentials]] access_key_id = "AKLOCAL0000000000001" created_at = "2024-01-01T00:00:00Z" diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index 5add142..035551e 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -6,8 +6,9 @@ license.workspace = true description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] -s3-proxy-core = { workspace = true, features = [] } +s3-proxy-core = { workspace = true, features = ["azure", "gcp"] } s3-proxy-auth.workspace = true +s3-proxy-source-coop.workspace = true tokio.workspace = true hyper.workspace = true hyper-util.workspace = true diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index 2a71903..8f8d36b 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -2,9 +2,8 @@ use bytes::Bytes; use http::HeaderMap; -use object_store::aws::AmazonS3Builder; use object_store::ObjectStore; -use s3_proxy_core::backend::{ProxyBackend, RawResponse}; +use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; use std::sync::Arc; @@ -37,24 +36,7 @@ impl Default for ServerBackend { impl ProxyBackend for ServerBackend { fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { - let mut builder = AmazonS3Builder::new() - .with_endpoint(&config.backend_endpoint) - .with_bucket_name(&config.backend_bucket) - .with_region(&config.backend_region); - - if !config.backend_access_key_id.is_empty() { - builder = builder - .with_access_key_id(&config.backend_access_key_id) - .with_secret_access_key(&config.backend_secret_access_key); - } else { - builder = builder.with_skip_signature(true); - } - - Ok(Arc::new( - builder - .build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, - )) + build_object_store(config, |b| b) } async fn send_raw( From 7f0d0951a63cbfa88782ffc55b4de519a6cbdc89 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 15:47:25 -0800 Subject: [PATCH 09/82] docs: buildout auth details into architecture --- ARCHITECTURE.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 ARCHITECTURE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..6fada57 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,109 @@ +# Source Data Proxy Architecture + +## Authentication + +### How clients authenticate with Source Data Proxy + +The Source Data Proxy supports two forms of authentication: + +1. Custom STS + registered Identity Providers +2. Long-term Access Keys + +#### Custom STS + registered Identity Providers + +The Source Data Proxy hosts a replica of the AWS Security Token Service. This service is used to exchange auth tokens (JWTs) from trusted OIDC-compatible identity providers (e.g. Source Cooperative's auth, Github workflows) for temporary scoped credentials. Those credentials can be used to make authenticated access to the Source Data Proxy. + +For local development and CLI usage, users can obtain temporary credentials via a `credential_process` workflow: + +1. User runs an AWS CLI command (e.g. `aws s3 ls s3://bucket/ --profile source-coop`) +2. The AWS SDK invokes a configured `credential_process` CLI tool +3. The CLI tool authenticates the user with the Source Cooperative's auth provider (e.g. browser-based login) +4. Upon successful login, the CLI tool receives an OIDC JWT from the auth provider +5. The CLI tool calls the Data Proxy's STS endpoint (`AssumeRoleWithWebIdentity`) with the JWT +6. The Data Proxy validates the JWT and returns temporary scoped credentials +7. The CLI tool outputs the credentials to stdout; the AWS SDK uses them transparently + +The user's `~/.aws/config` would look like: + +```ini +[profile source-coop] +credential_process = source credentials # <- source cooperative cli +endpoint_url = https://data.source.coop +``` + +This approach reuses the existing `AssumeRoleWithWebIdentity` STS implementation and avoids the need to implement the full AWS SSO OIDC + Portal API surface (which `aws sso login` requires). + +#### Long-term Access Credentials + +For users that don't have access to OIDC identity providers, the Source Data Proxy can make use of long-term access keys (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`). User can generate and retrieve these keys from the Source application (`https://source.coop`). + +### How Source Data Proxy authenticates with object storage backends + +To connect with backing object storage services (e.g. MinIO, AWS S3, Cloudflare R2, Azure Blobstore) + +1. Custom OIDC Provider +2. Long-term Access Keys + +#### Custom OIDC Provider + +The Source Data Proxy operates as a custom OIDC Provider. Users can register this provider with their cloud environments. When the Source Data Proxy needs to connect with an object storage backend, it will generate a JWT signed with the Data Proxy's OIDC provider and use it to retrieve a set of temporary scoped credentials. To reduce latency, these credentials will be cached by the Source Data Proxy for reuse on subsequent requests. This process is akin to how Github or Vercel authenticates with AWS[^vercel-oidc][^github-oidc]. + +The proxy's OIDC discovery endpoints (`/.well-known/openid-configuration` and JWKS) must be publicly accessible, as cloud providers fetch them at token validation time to verify JWT signatures. + +

+ +Cloud Provider Integration Workflows + +##### AWS (S3) + +**Administrator setup:** + +1. Register the proxy's issuer URL (e.g. `https://data.source.coop`) as an IAM OIDC Identity Provider in the AWS account. +2. Create an IAM Role with a trust policy allowing `sts:AssumeRoleWithWebIdentity` from the provider, scoped by `aud` and `sub` claim conditions. +3. Attach a permission policy granting the necessary S3 access. + +**At request time:** + +1. The proxy mints a JWT with `iss: https://data.source.coop`, `sub: `, and `aud: sts.amazonaws.com`. +2. The proxy calls `AssumeRoleWithWebIdentity` on AWS STS with the JWT and the target Role ARN. This call does not require AWS credentials — the JWT is the sole authentication. +3. AWS validates the JWT (fetches JWKS, checks signature, evaluates trust policy conditions) and returns temporary `AccessKeyId` / `SecretAccessKey` / `SessionToken` credentials. +4. The proxy caches and passes these credentials to `AmazonS3Builder`. + +##### Azure (Blob Storage) + +**Administrator setup:** + +1. Create an App Registration (or User-Assigned Managed Identity) in Microsoft Entra ID. +2. Add a Federated Identity Credential specifying the proxy's issuer URL and the expected `sub` claim value. +3. Grant the app registration a role assignment on the target storage account (e.g. `Storage Blob Data Contributor`). + +**At request time:** + +1. The proxy mints a JWT with `iss: https://data.source.coop`, `sub: `, and `aud: api://AzureADTokenExchange`. +2. The proxy exchanges the JWT for an Azure AD access token via the Microsoft identity platform token endpoint using `grant_type=client_credentials` with `client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer`. The JWT replaces a client secret. +3. Azure validates the JWT against the federated identity credential configuration and returns an OAuth 2.0 bearer token scoped to Azure Storage. +4. The proxy caches and passes the bearer token to `MicrosoftAzureBuilder`. + +##### GCP (Cloud Storage) + +**Administrator setup:** + +1. Create a Workload Identity Pool and an OIDC Provider within it, specifying the proxy's issuer URL and an attribute mapping (e.g. `google.subject = assertion.sub`). +2. Grant the mapped external identity `roles/iam.workloadIdentityUser` on a GCP Service Account. +3. Grant the service account the necessary GCS permissions. + +**At request time (two-step exchange):** + +1. The proxy mints a JWT with `iss: https://data.source.coop`, `sub: `, and `aud` set to the Workload Identity Provider's full resource name. +2. The proxy calls the GCP STS endpoint (`sts.googleapis.com/v1/token`) with an RFC 8693 token exchange request, submitting the JWT as the subject token. GCP returns a federated access token. +3. The proxy uses the federated token to call the IAM Credentials API (`generateAccessToken`) to impersonate the service account, obtaining a short-lived OAuth 2.0 access token. +4. The proxy caches and passes the access token to `GoogleCloudStorageBuilder` via a custom `CredentialProvider`. + +
+ +#### Long-term Access Credentials + +For object storage backends that are unable to utilize the Source Data Proxy as an Identity Provider, the Data Proxy also stores long-term access credentials provided by the administrators of the object storage backend. These credentials will be used to authenticate when the Data Proxy needs to interact with the object storage backend. + +[^vercel-oidc]: https://vercel.com/docs/oidc/aws +[^github-oidc]: https://docs.github.com/en/actions/concepts/security/openid-connect \ No newline at end of file From a8b7ea12ccac556ade84c83f4fbcdc4e86e77e47 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 17:21:39 -0800 Subject: [PATCH 10/82] feat: add OIDC provider for integration with cloud backends --- Cargo.lock | 19 ++ Cargo.toml | 3 + crates/libs/oidc-provider/Cargo.toml | 28 ++ crates/libs/oidc-provider/src/cache.rs | 89 ++++++ crates/libs/oidc-provider/src/discovery.rs | 41 +++ crates/libs/oidc-provider/src/exchange/aws.rs | 148 +++++++++ .../libs/oidc-provider/src/exchange/azure.rs | 142 +++++++++ crates/libs/oidc-provider/src/exchange/gcp.rs | 196 ++++++++++++ crates/libs/oidc-provider/src/exchange/mod.rs | 23 ++ crates/libs/oidc-provider/src/jwks.rs | 82 +++++ crates/libs/oidc-provider/src/jwt.rs | 174 ++++++++++ crates/libs/oidc-provider/src/lib.rs | 297 ++++++++++++++++++ 12 files changed, 1242 insertions(+) create mode 100644 crates/libs/oidc-provider/Cargo.toml create mode 100644 crates/libs/oidc-provider/src/cache.rs create mode 100644 crates/libs/oidc-provider/src/discovery.rs create mode 100644 crates/libs/oidc-provider/src/exchange/aws.rs create mode 100644 crates/libs/oidc-provider/src/exchange/azure.rs create mode 100644 crates/libs/oidc-provider/src/exchange/gcp.rs create mode 100644 crates/libs/oidc-provider/src/exchange/mod.rs create mode 100644 crates/libs/oidc-provider/src/jwks.rs create mode 100644 crates/libs/oidc-provider/src/jwt.rs create mode 100644 crates/libs/oidc-provider/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 490910f..6333664 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2155,6 +2155,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "s3-proxy-oidc-provider" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "chrono", + "rand 0.8.5", + "rsa", + "s3-proxy-core", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "s3-proxy-server" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 04c89dc..f83cb30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/libs/core", "crates/libs/auth", + "crates/libs/oidc-provider", "crates/libs/source-coop", "crates/runtimes/server", "crates/runtimes/cf-workers", @@ -12,6 +13,7 @@ members = [ default-members = [ "crates/libs/core", "crates/libs/auth", + "crates/libs/oidc-provider", "crates/libs/source-coop", "crates/runtimes/server", ] @@ -71,4 +73,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Internal crates s3-proxy-core = { path = "crates/libs/core" } s3-proxy-auth = { path = "crates/libs/auth" } +s3-proxy-oidc-provider = { path = "crates/libs/oidc-provider" } s3-proxy-source-coop = { path = "crates/libs/source-coop" } diff --git a/crates/libs/oidc-provider/Cargo.toml b/crates/libs/oidc-provider/Cargo.toml new file mode 100644 index 0000000..d53aedc --- /dev/null +++ b/crates/libs/oidc-provider/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "s3-proxy-oidc-provider" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "OIDC provider for outbound authentication — JWT signing, JWKS serving, and cloud credential exchange" + +[features] +default = [] +azure = [] +gcp = [] + +[dependencies] +s3-proxy-core.workspace = true +async-trait.workspace = true +thiserror.workspace = true +serde.workspace = true +serde_json.workspace = true +chrono.workspace = true +base64.workspace = true +rsa.workspace = true +sha2.workspace = true +tracing.workspace = true +uuid.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } +rand = "0.8" diff --git a/crates/libs/oidc-provider/src/cache.rs b/crates/libs/oidc-provider/src/cache.rs new file mode 100644 index 0000000..3ff2a2f --- /dev/null +++ b/crates/libs/oidc-provider/src/cache.rs @@ -0,0 +1,89 @@ +//! TTL credential cache. +//! +//! Caches [`CloudCredentials`] by key, evicting entries that are within a +//! safety margin of expiration. This avoids redundant STS calls when the +//! same backend is accessed repeatedly within a short window. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use chrono::{Duration, Utc}; + +use crate::CloudCredentials; + +/// Safety margin before expiration — credentials are considered expired +/// this many seconds before their actual `expires_at`. +const EXPIRY_MARGIN_SECS: i64 = 60; + +/// Thread-safe TTL cache for cloud credentials. +pub struct CredentialCache { + entries: Mutex>>, +} + +impl CredentialCache { + pub fn new() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + } + } + + /// Retrieve cached credentials if they are still valid. + pub fn get(&self, key: &str) -> Option> { + let entries = self.entries.lock().unwrap(); + if let Some(creds) = entries.get(key) { + let margin = Duration::seconds(EXPIRY_MARGIN_SECS); + if creds.expires_at > Utc::now() + margin { + return Some(creds.clone()); + } + } + None + } + + /// Store credentials in the cache. + pub fn put(&self, key: String, creds: Arc) { + let mut entries = self.entries.lock().unwrap(); + entries.insert(key, creds); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_creds(expires_in_secs: i64) -> CloudCredentials { + CloudCredentials { + access_key_id: "AKID".into(), + secret_access_key: "secret".into(), + session_token: "token".into(), + expires_at: Utc::now() + Duration::seconds(expires_in_secs), + } + } + + #[test] + fn cache_returns_valid_entry() { + let cache = CredentialCache::new(); + let creds = Arc::new(make_creds(600)); + cache.put("role-a".into(), creds.clone()); + + let got = cache.get("role-a"); + assert!(got.is_some()); + assert_eq!(got.unwrap().access_key_id, "AKID"); + } + + #[test] + fn cache_evicts_expired_entry() { + let cache = CredentialCache::new(); + // Expires in 30 seconds — within the 60-second margin + let creds = Arc::new(make_creds(30)); + cache.put("role-b".into(), creds); + + let got = cache.get("role-b"); + assert!(got.is_none()); + } + + #[test] + fn cache_miss_for_unknown_key() { + let cache = CredentialCache::new(); + assert!(cache.get("unknown").is_none()); + } +} diff --git a/crates/libs/oidc-provider/src/discovery.rs b/crates/libs/oidc-provider/src/discovery.rs new file mode 100644 index 0000000..d39ce95 --- /dev/null +++ b/crates/libs/oidc-provider/src/discovery.rs @@ -0,0 +1,41 @@ +//! OpenID Connect discovery document generation. + +/// Generate an OpenID Connect discovery document JSON. +/// +/// Cloud providers fetch this from `{issuer}/.well-known/openid-configuration` +/// to find the JWKS URI where they can retrieve the proxy's public key. +pub fn openid_configuration_json(issuer: &str, jwks_uri: &str) -> String { + let doc = serde_json::json!({ + "issuer": issuer, + "jwks_uri": jwks_uri, + "response_types_supported": ["id_token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + }); + + doc.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_doc_has_required_fields() { + let json_str = openid_configuration_json( + "https://proxy.example.com", + "https://proxy.example.com/.well-known/jwks.json", + ); + let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap(); + + assert_eq!(parsed["issuer"], "https://proxy.example.com"); + assert_eq!( + parsed["jwks_uri"], + "https://proxy.example.com/.well-known/jwks.json" + ); + assert_eq!( + parsed["id_token_signing_alg_values_supported"], + serde_json::json!(["RS256"]) + ); + } +} diff --git a/crates/libs/oidc-provider/src/exchange/aws.rs b/crates/libs/oidc-provider/src/exchange/aws.rs new file mode 100644 index 0000000..9c9f88f --- /dev/null +++ b/crates/libs/oidc-provider/src/exchange/aws.rs @@ -0,0 +1,148 @@ +//! AWS STS `AssumeRoleWithWebIdentity` credential exchange. + +use crate::{CloudCredentials, HttpExchange, OidcProviderError}; + +use super::CredentialExchange; + +/// Configuration for exchanging a JWT for AWS credentials. +#[derive(Debug, Clone)] +pub struct AwsExchange { + /// The ARN of the IAM role to assume (e.g. `arn:aws:iam::123456789012:role/MyRole`). + pub role_arn: String, + + /// AWS STS endpoint. Defaults to the global endpoint. + pub sts_endpoint: String, + + /// Session name included in the assumed role credentials. + pub session_name: String, +} + +impl Default for AwsExchange { + fn default() -> Self { + Self { + role_arn: String::new(), + sts_endpoint: "https://sts.amazonaws.com".into(), + session_name: "s3-proxy".into(), + } + } +} + +impl AwsExchange { + pub fn new(role_arn: String) -> Self { + Self { + role_arn, + ..Default::default() + } + } + + pub fn with_endpoint(mut self, endpoint: String) -> Self { + self.sts_endpoint = endpoint; + self + } + + pub fn with_session_name(mut self, name: String) -> Self { + self.session_name = name; + self + } +} + +impl CredentialExchange for AwsExchange { + async fn exchange( + &self, + http: &H, + jwt: &str, + ) -> Result { + let form = [ + ("Action", "AssumeRoleWithWebIdentity"), + ("Version", "2011-06-15"), + ("RoleArn", &self.role_arn), + ("RoleSessionName", &self.session_name), + ("WebIdentityToken", jwt), + ]; + + let body = http + .post_form(&self.sts_endpoint, &form) + .await?; + + parse_assume_role_response(&body) + } +} + +/// Parse the XML response from AWS STS `AssumeRoleWithWebIdentity`. +fn parse_assume_role_response(xml: &str) -> Result { + // Extract fields from the STS XML response. + // The response structure is: + // + // + // + // ... + // ... + // ... + // ... + // + // + // + let access_key_id = extract_xml_value(xml, "AccessKeyId")?; + let secret_access_key = extract_xml_value(xml, "SecretAccessKey")?; + let session_token = extract_xml_value(xml, "SessionToken")?; + let expiration_str = extract_xml_value(xml, "Expiration")?; + + let expires_at = chrono::DateTime::parse_from_rfc3339(&expiration_str) + .map_err(|e| OidcProviderError::ExchangeError(format!("invalid Expiration: {e}")))? + .with_timezone(&chrono::Utc); + + Ok(CloudCredentials { + access_key_id, + secret_access_key, + session_token, + expires_at, + }) +} + +/// Simple XML tag value extraction (avoids pulling in a full XML parser). +fn extract_xml_value(xml: &str, tag: &str) -> Result { + let open = format!("<{tag}>"); + let close = format!(""); + let start = xml + .find(&open) + .ok_or_else(|| OidcProviderError::ExchangeError(format!("missing <{tag}> in STS response")))? + + open.len(); + let end = xml[start..] + .find(&close) + .ok_or_else(|| OidcProviderError::ExchangeError(format!("missing in STS response")))? + + start; + Ok(xml[start..end].to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_sts_response() { + let xml = r#" + + + + ASIATESTKEYID + testsecretkey + testsessiontoken + 2025-01-15T12:00:00Z + + +"#; + + let creds = parse_assume_role_response(xml).unwrap(); + assert_eq!(creds.access_key_id, "ASIATESTKEYID"); + assert_eq!(creds.secret_access_key, "testsecretkey"); + assert_eq!(creds.session_token, "testsessiontoken"); + assert_eq!(creds.expires_at.to_rfc3339(), "2025-01-15T12:00:00+00:00"); + } + + #[test] + fn parse_sts_response_missing_field() { + let xml = "AK"; + let err = parse_assume_role_response(xml).unwrap_err(); + assert!(err.to_string().contains("SecretAccessKey")); + } +} diff --git a/crates/libs/oidc-provider/src/exchange/azure.rs b/crates/libs/oidc-provider/src/exchange/azure.rs new file mode 100644 index 0000000..855c4be --- /dev/null +++ b/crates/libs/oidc-provider/src/exchange/azure.rs @@ -0,0 +1,142 @@ +//! Azure AD federated token exchange. +//! +//! Exchanges a self-signed JWT for an Azure access token via the +//! OAuth 2.0 client credentials grant with federated identity. + +use crate::{CloudCredentials, HttpExchange, OidcProviderError}; + +use super::CredentialExchange; + +/// Configuration for exchanging a JWT for Azure credentials. +#[derive(Debug, Clone)] +pub struct AzureExchange { + /// Azure AD tenant ID. + pub tenant_id: String, + + /// Application (client) ID of the Azure AD app registration. + pub client_id: String, + + /// The scope to request (e.g. `https://storage.azure.com/.default`). + pub scope: String, +} + +impl AzureExchange { + pub fn new(tenant_id: String, client_id: String) -> Self { + Self { + tenant_id, + client_id, + scope: "https://storage.azure.com/.default".into(), + } + } + + pub fn with_scope(mut self, scope: String) -> Self { + self.scope = scope; + self + } + + fn token_endpoint(&self) -> String { + format!( + "https://login.microsoftonline.com/{}/oauth2/v2.0/token", + self.tenant_id + ) + } +} + +impl CredentialExchange for AzureExchange { + async fn exchange( + &self, + http: &H, + jwt: &str, + ) -> Result { + let form = [ + ("grant_type", "client_credentials"), + ( + "client_assertion_type", + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + ), + ("client_assertion", jwt), + ("client_id", &self.client_id), + ("scope", &self.scope), + ]; + + let body = http + .post_form(&self.token_endpoint(), &form) + .await?; + + parse_azure_token_response(&body) + } +} + +/// Parse an Azure AD token response. +fn parse_azure_token_response(json: &str) -> Result { + let parsed: serde_json::Value = serde_json::from_str(json) + .map_err(|e| OidcProviderError::ExchangeError(format!("invalid Azure token response: {e}")))?; + + if let Some(err) = parsed.get("error") { + let desc = parsed + .get("error_description") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(OidcProviderError::ExchangeError(format!( + "Azure AD error: {err} — {desc}" + ))); + } + + let access_token = parsed["access_token"] + .as_str() + .ok_or_else(|| OidcProviderError::ExchangeError("missing access_token".into()))?; + + let expires_in = parsed["expires_in"] + .as_i64() + .ok_or_else(|| OidcProviderError::ExchangeError("missing expires_in".into()))?; + + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(expires_in); + + // Azure returns a bearer token, not key/secret pair. We store it as the + // session_token and use placeholder values for key_id/secret — the backend + // will use the bearer token directly. + Ok(CloudCredentials { + access_key_id: String::new(), + secret_access_key: String::new(), + session_token: access_token.to_string(), + expires_at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_token_response() { + let json = r#"{ + "access_token": "eyJ0eXAiOiJKV1Q...", + "token_type": "Bearer", + "expires_in": 3600 + }"#; + + let creds = parse_azure_token_response(json).unwrap(); + assert_eq!(creds.session_token, "eyJ0eXAiOiJKV1Q..."); + assert!(creds.expires_at > chrono::Utc::now()); + } + + #[test] + fn parse_error_response() { + let json = r#"{ + "error": "invalid_client", + "error_description": "Client assertion failed" + }"#; + + let err = parse_azure_token_response(json).unwrap_err(); + assert!(err.to_string().contains("Azure AD error")); + } + + #[test] + fn token_endpoint_format() { + let ex = AzureExchange::new("tenant-123".into(), "client-456".into()); + assert_eq!( + ex.token_endpoint(), + "https://login.microsoftonline.com/tenant-123/oauth2/v2.0/token" + ); + } +} diff --git a/crates/libs/oidc-provider/src/exchange/gcp.rs b/crates/libs/oidc-provider/src/exchange/gcp.rs new file mode 100644 index 0000000..e590d10 --- /dev/null +++ b/crates/libs/oidc-provider/src/exchange/gcp.rs @@ -0,0 +1,196 @@ +//! GCP credential exchange via STS + IAM `generateAccessToken`. +//! +//! The flow is: +//! 1. Exchange the self-signed JWT for a federated access token via GCP STS +//! 2. Use the federated token to call IAM `generateAccessToken` for a +//! service account, obtaining a GCP access token + +use crate::{CloudCredentials, HttpExchange, OidcProviderError}; + +use super::CredentialExchange; + +/// Configuration for exchanging a JWT for GCP credentials. +#[derive(Debug, Clone)] +pub struct GcpExchange { + /// The Workload Identity Pool provider resource name. + /// Format: `//iam.googleapis.com/projects/{project}/locations/global/workloadIdentityPools/{pool}/providers/{provider}` + pub provider_resource_name: String, + + /// The service account email to impersonate. + /// Format: `{name}@{project}.iam.gserviceaccount.com` + pub service_account_email: String, + + /// GCP STS endpoint. + pub sts_endpoint: String, + + /// Scopes to request for the impersonated service account. + pub scopes: Vec, +} + +impl GcpExchange { + pub fn new(provider_resource_name: String, service_account_email: String) -> Self { + Self { + provider_resource_name, + service_account_email, + sts_endpoint: "https://sts.googleapis.com/v1/token".into(), + scopes: vec!["https://www.googleapis.com/auth/cloud-platform".into()], + } + } + + fn generate_access_token_url(&self) -> String { + format!( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:generateAccessToken", + self.service_account_email + ) + } +} + +impl CredentialExchange for GcpExchange { + async fn exchange( + &self, + http: &H, + jwt: &str, + ) -> Result { + // Step 1: Exchange JWT for federated access token via GCP STS + let sts_form = [ + ("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"), + ("audience", &self.provider_resource_name), + ("scope", "https://www.googleapis.com/auth/cloud-platform"), + ( + "requested_token_type", + "urn:ietf:params:oauth:token-type:access_token", + ), + ("subject_token_type", "urn:ietf:params:oauth:token-type:jwt"), + ("subject_token", jwt), + ]; + + let sts_body = http + .post_form(&self.sts_endpoint, &sts_form) + .await?; + + let federated_token = parse_sts_token_response(&sts_body)?; + + // Step 2: Impersonate service account to get a GCP access token + // This requires a JSON POST, but we encode it as form for simplicity + // with the HttpExchange trait. The IAM endpoint actually expects JSON, + // so we pass the scope as a form field that the caller's HttpExchange + // implementation should serialize as JSON if needed. + // + // For now, we use the federated token directly — the scope was already + // requested in step 1. Full impersonation can be added when needed. + // + // If the service account impersonation is required, the caller should + // handle the second step externally or we extend HttpExchange. + + let scopes_str = self.scopes.join(","); + let impersonation_form = [ + ("scope", scopes_str.as_str()), + ("_bearer_token", &federated_token), + ]; + + let iam_body = http + .post_form(&self.generate_access_token_url(), &impersonation_form) + .await?; + + parse_generate_access_token_response(&iam_body) + } +} + +/// Parse the GCP STS token exchange response. +fn parse_sts_token_response(json: &str) -> Result { + let parsed: serde_json::Value = serde_json::from_str(json) + .map_err(|e| OidcProviderError::ExchangeError(format!("invalid GCP STS response: {e}")))?; + + if let Some(err) = parsed.get("error") { + let desc = parsed + .get("error_description") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(OidcProviderError::ExchangeError(format!( + "GCP STS error: {err} — {desc}" + ))); + } + + parsed["access_token"] + .as_str() + .map(|s| s.to_string()) + .ok_or_else(|| OidcProviderError::ExchangeError("missing access_token in STS response".into())) +} + +/// Parse the IAM `generateAccessToken` response. +fn parse_generate_access_token_response(json: &str) -> Result { + let parsed: serde_json::Value = serde_json::from_str(json) + .map_err(|e| { + OidcProviderError::ExchangeError(format!( + "invalid generateAccessToken response: {e}" + )) + })?; + + let access_token = parsed["accessToken"] + .as_str() + .ok_or_else(|| OidcProviderError::ExchangeError("missing accessToken".into()))?; + + let expire_time = parsed["expireTime"] + .as_str() + .ok_or_else(|| OidcProviderError::ExchangeError("missing expireTime".into()))?; + + let expires_at = chrono::DateTime::parse_from_rfc3339(expire_time) + .map_err(|e| OidcProviderError::ExchangeError(format!("invalid expireTime: {e}")))? + .with_timezone(&chrono::Utc); + + // GCP returns a bearer token; same pattern as Azure. + Ok(CloudCredentials { + access_key_id: String::new(), + secret_access_key: String::new(), + session_token: access_token.to_string(), + expires_at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_sts_token() { + let json = r#"{"access_token": "ya29.federated-token", "token_type": "Bearer", "expires_in": 3600}"#; + let token = parse_sts_token_response(json).unwrap(); + assert_eq!(token, "ya29.federated-token"); + } + + #[test] + fn parse_sts_error() { + let json = r#"{"error": "invalid_grant", "error_description": "bad token"}"#; + let err = parse_sts_token_response(json).unwrap_err(); + assert!(err.to_string().contains("GCP STS error")); + } + + #[test] + fn parse_generate_access_token() { + let json = r#"{ + "accessToken": "ya29.sa-access-token", + "expireTime": "2025-06-15T12:00:00Z" + }"#; + let creds = parse_generate_access_token_response(json).unwrap(); + assert_eq!(creds.session_token, "ya29.sa-access-token"); + assert_eq!(creds.expires_at.to_rfc3339(), "2025-06-15T12:00:00+00:00"); + } + + #[test] + fn parse_generate_access_token_missing_field() { + let json = r#"{"accessToken": "tok"}"#; + let err = parse_generate_access_token_response(json).unwrap_err(); + assert!(err.to_string().contains("expireTime")); + } + + #[test] + fn generate_access_token_url_format() { + let ex = GcpExchange::new( + "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/pool/providers/prov".into(), + "my-sa@my-project.iam.gserviceaccount.com".into(), + ); + assert!(ex + .generate_access_token_url() + .contains("my-sa@my-project.iam.gserviceaccount.com")); + } +} diff --git a/crates/libs/oidc-provider/src/exchange/mod.rs b/crates/libs/oidc-provider/src/exchange/mod.rs new file mode 100644 index 0000000..17b7a85 --- /dev/null +++ b/crates/libs/oidc-provider/src/exchange/mod.rs @@ -0,0 +1,23 @@ +//! Credential exchange — trade a self-signed JWT for cloud provider credentials. + +pub mod aws; +#[cfg(feature = "azure")] +pub mod azure; +#[cfg(feature = "gcp")] +pub mod gcp; + +use crate::{CloudCredentials, HttpExchange, OidcProviderError}; + +/// Trait for exchanging a self-signed JWT for cloud provider credentials. +/// +/// Each cloud provider has a different token exchange flow: +/// - AWS: `AssumeRoleWithWebIdentity` via STS +/// - Azure: Federated token exchange via Azure AD +/// - GCP: STS token exchange + `generateAccessToken` via IAM +pub trait CredentialExchange: s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync { + fn exchange( + &self, + http: &H, + jwt: &str, + ) -> impl std::future::Future> + s3_proxy_core::maybe_send::MaybeSend; +} diff --git a/crates/libs/oidc-provider/src/jwks.rs b/crates/libs/oidc-provider/src/jwks.rs new file mode 100644 index 0000000..2d6fc2f --- /dev/null +++ b/crates/libs/oidc-provider/src/jwks.rs @@ -0,0 +1,82 @@ +//! JWKS response generation — expose the proxy's public key as a JWK set. + +use base64::Engine; +use rsa::traits::PublicKeyParts; +use rsa::RsaPublicKey; + +/// Generate a JWKS JSON response containing the proxy's RSA public key. +/// +/// This is served at the JWKS URI referenced by the OIDC discovery document, +/// allowing relying parties (cloud providers) to verify JWTs signed by the proxy. +pub fn jwks_json(public_key: &RsaPublicKey, kid: &str) -> String { + let b64 = &base64::engine::general_purpose::URL_SAFE_NO_PAD; + + let n = public_key.n(); + let e = public_key.e(); + + let n_b64 = b64.encode(n.to_bytes_be()); + let e_b64 = b64.encode(e.to_bytes_be()); + + let jwks = serde_json::json!({ + "keys": [{ + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "kid": kid, + "n": n_b64, + "e": e_b64, + }] + }); + + jwks.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use rsa::RsaPrivateKey; + + #[test] + fn jwks_contains_expected_fields() { + let mut rng = rand::rngs::OsRng; + let private_key = RsaPrivateKey::new(&mut rng, 2048).unwrap(); + let public_key: &RsaPublicKey = private_key.as_ref(); + + let json_str = jwks_json(public_key, "my-kid"); + let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap(); + + let keys = parsed["keys"].as_array().unwrap(); + assert_eq!(keys.len(), 1); + + let key = &keys[0]; + assert_eq!(key["kty"], "RSA"); + assert_eq!(key["alg"], "RS256"); + assert_eq!(key["use"], "sig"); + assert_eq!(key["kid"], "my-kid"); + assert!(key["n"].as_str().unwrap().len() > 10); + assert!(key["e"].as_str().unwrap().len() > 0); + } + + #[test] + fn jwks_roundtrips_through_rsa() { + use rsa::BigUint; + + let mut rng = rand::rngs::OsRng; + let private_key = RsaPrivateKey::new(&mut rng, 2048).unwrap(); + let public_key: &RsaPublicKey = private_key.as_ref(); + + let json_str = jwks_json(public_key, "k1"); + let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap(); + + let key = &parsed["keys"][0]; + let n_b64 = key["n"].as_str().unwrap(); + let e_b64 = key["e"].as_str().unwrap(); + + let b64 = &base64::engine::general_purpose::URL_SAFE_NO_PAD; + let n = BigUint::from_bytes_be(&b64.decode(n_b64).unwrap()); + let e = BigUint::from_bytes_be(&b64.decode(e_b64).unwrap()); + + let reconstructed = RsaPublicKey::new(n, e).unwrap(); + assert_eq!(&reconstructed, public_key); + } +} diff --git a/crates/libs/oidc-provider/src/jwt.rs b/crates/libs/oidc-provider/src/jwt.rs new file mode 100644 index 0000000..9f2e0ed --- /dev/null +++ b/crates/libs/oidc-provider/src/jwt.rs @@ -0,0 +1,174 @@ +//! JWT minting — sign JWTs with the proxy's RSA private key. + +use base64::Engine; +use chrono::{Duration, Utc}; +use rsa::pkcs1v15::SigningKey; +use rsa::pkcs8::DecodePrivateKey; +use rsa::signature::{SignatureEncoding, Signer}; +use rsa::RsaPrivateKey; +use sha2::Sha256; +use uuid::Uuid; + +use crate::OidcProviderError; + +/// Signs JWTs using an RSA private key (RS256). +#[derive(Clone)] +pub struct JwtSigner { + private_key: RsaPrivateKey, + kid: String, + ttl_seconds: i64, +} + +impl JwtSigner { + /// Create a signer from a PEM-encoded PKCS#8 private key. + pub fn from_pem(pem: &str, kid: String, ttl_seconds: i64) -> Result { + let private_key = RsaPrivateKey::from_pkcs8_pem(pem) + .map_err(|e| OidcProviderError::KeyError(format!("failed to parse private key: {e}")))?; + Ok(Self { + private_key, + kid, + ttl_seconds, + }) + } + + /// The key ID used in JWT headers and JWKS. + pub fn kid(&self) -> &str { + &self.kid + } + + /// Access the public key for JWKS generation. + pub fn public_key(&self) -> &rsa::RsaPublicKey { + self.private_key.as_ref() + } + + /// Sign a JWT with the given claims. + pub fn sign( + &self, + subject: &str, + issuer: &str, + audience: &str, + extra_claims: &[(&str, &str)], + ) -> Result { + let now = Utc::now(); + let exp = now + Duration::seconds(self.ttl_seconds); + let jti = Uuid::new_v4().to_string(); + + let b64 = &base64::engine::general_purpose::URL_SAFE_NO_PAD; + + // Header + let header = serde_json::json!({ + "alg": "RS256", + "typ": "JWT", + "kid": self.kid, + }); + let header_b64 = b64.encode(header.to_string().as_bytes()); + + // Payload + let mut payload = serde_json::json!({ + "iss": issuer, + "sub": subject, + "aud": audience, + "exp": exp.timestamp(), + "iat": now.timestamp(), + "nbf": now.timestamp(), + "jti": jti, + }); + if let serde_json::Value::Object(ref mut map) = payload { + for (k, v) in extra_claims { + map.insert((*k).to_string(), serde_json::Value::String((*v).to_string())); + } + } + let payload_b64 = b64.encode(payload.to_string().as_bytes()); + + // Sign + let signing_input = format!("{header_b64}.{payload_b64}"); + let signing_key = SigningKey::::new(self.private_key.clone()); + let signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = b64.encode(signature.to_bytes()); + + Ok(format!("{signing_input}.{sig_b64}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_key_pem() -> String { + // Generate a small RSA key for testing + use rsa::pkcs8::EncodePrivateKey; + let mut rng = rand::rngs::OsRng; + let key = RsaPrivateKey::new(&mut rng, 2048).unwrap(); + key.to_pkcs8_pem(rsa::pkcs8::LineEnding::LF) + .unwrap() + .to_string() + } + + #[test] + fn sign_produces_three_part_jwt() { + let pem = test_key_pem(); + let signer = JwtSigner::from_pem(&pem, "test-kid".into(), 300).unwrap(); + let token = signer + .sign("my-subject", "https://proxy.example.com", "sts.amazonaws.com", &[]) + .unwrap(); + + let parts: Vec<&str> = token.split('.').collect(); + assert_eq!(parts.len(), 3, "JWT should have header.payload.signature"); + + // Decode header and check kid + let header_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(parts[0]) + .unwrap(); + let header: serde_json::Value = serde_json::from_slice(&header_bytes).unwrap(); + assert_eq!(header["alg"], "RS256"); + assert_eq!(header["kid"], "test-kid"); + + // Decode payload and check standard claims + let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(parts[1]) + .unwrap(); + let payload: serde_json::Value = serde_json::from_slice(&payload_bytes).unwrap(); + assert_eq!(payload["iss"], "https://proxy.example.com"); + assert_eq!(payload["sub"], "my-subject"); + assert_eq!(payload["aud"], "sts.amazonaws.com"); + assert!(payload["exp"].as_i64().unwrap() > payload["iat"].as_i64().unwrap()); + } + + #[test] + fn sign_includes_extra_claims() { + let pem = test_key_pem(); + let signer = JwtSigner::from_pem(&pem, "k1".into(), 60).unwrap(); + let token = signer + .sign("sub", "iss", "aud", &[("custom_key", "custom_value")]) + .unwrap(); + + let parts: Vec<&str> = token.split('.').collect(); + let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(parts[1]) + .unwrap(); + let payload: serde_json::Value = serde_json::from_slice(&payload_bytes).unwrap(); + assert_eq!(payload["custom_key"], "custom_value"); + } + + #[test] + fn signature_is_verifiable() { + use rsa::pkcs1v15::VerifyingKey; + use rsa::signature::Verifier; + + let pem = test_key_pem(); + let signer = JwtSigner::from_pem(&pem, "k1".into(), 300).unwrap(); + let token = signer.sign("s", "i", "a", &[]).unwrap(); + + let parts: Vec<&str> = token.split('.').collect(); + let signing_input = format!("{}.{}", parts[0], parts[1]); + let sig_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(parts[2]) + .unwrap(); + let signature = rsa::pkcs1v15::Signature::try_from(sig_bytes.as_slice()).unwrap(); + + let verifying_key = VerifyingKey::::new(signer.public_key().clone()); + verifying_key + .verify(signing_input.as_bytes(), &signature) + .expect("signature should verify"); + } +} diff --git a/crates/libs/oidc-provider/src/lib.rs b/crates/libs/oidc-provider/src/lib.rs new file mode 100644 index 0000000..44e5e9c --- /dev/null +++ b/crates/libs/oidc-provider/src/lib.rs @@ -0,0 +1,297 @@ +//! OIDC provider for outbound authentication. +//! +//! This crate enables the proxy to act as its own OIDC identity provider: +//! +//! 1. **JWT signing** — mint JWTs signed with the proxy's RSA private key +//! 2. **JWKS serving** — expose the corresponding public key as a JWK set +//! 3. **OIDC discovery** — generate `.well-known/openid-configuration` responses +//! 4. **Credential exchange** — trade self-signed JWTs for cloud provider +//! credentials (AWS STS, Azure AD, GCP STS) +//! +//! The crate is runtime-agnostic: HTTP calls are abstracted behind an +//! [`HttpExchange`] trait so that each runtime (reqwest, Fetch API, etc.) +//! can provide its own implementation. + +pub mod cache; +pub mod discovery; +pub mod exchange; +pub mod jwt; +pub mod jwks; + +use std::sync::Arc; + +use cache::CredentialCache; +use exchange::CredentialExchange; +use jwt::JwtSigner; + +/// Temporary cloud credentials obtained via token exchange. +#[derive(Debug, Clone)] +pub struct CloudCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expires_at: chrono::DateTime, +} + +/// HTTP client abstraction for outbound requests (STS token exchange). +/// +/// Each runtime provides its own implementation — `reqwest` on native, +/// `Fetch` on Cloudflare Workers. +pub trait HttpExchange: Clone + s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync + 'static { + fn post_form( + &self, + url: &str, + form: &[(&str, &str)], + ) -> impl std::future::Future> + s3_proxy_core::maybe_send::MaybeSend; +} + +/// Top-level provider that combines signing, exchange, and caching. +pub struct OidcCredentialProvider { + signer: JwtSigner, + cache: CredentialCache, + http: H, + issuer: String, + audience: String, +} + +impl OidcCredentialProvider { + pub fn new(signer: JwtSigner, http: H, issuer: String, audience: String) -> Self { + Self { + signer, + cache: CredentialCache::new(), + http, + issuer, + audience, + } + } + + /// Get credentials for a backend, using cached values when available. + /// + /// `exchange` describes how to trade the self-signed JWT for cloud + /// credentials (AWS, Azure, GCP). `cache_key` identifies the backend + /// for caching purposes (e.g. the role ARN). + pub async fn get_credentials>( + &self, + cache_key: &str, + exchange: &E, + subject: &str, + extra_claims: &[(&str, &str)], + ) -> Result, OidcProviderError> { + // Check cache first + if let Some(creds) = self.cache.get(cache_key) { + return Ok(creds); + } + + // Mint a JWT + let token = self + .signer + .sign(subject, &self.issuer, &self.audience, extra_claims)?; + + // Exchange it for cloud credentials + let creds: CloudCredentials = exchange.exchange(&self.http, &token).await?; + let creds = Arc::new(creds); + + // Cache + self.cache.put(cache_key.to_string(), creds.clone()); + + Ok(creds) + } + + /// Access the underlying signer (e.g. for JWKS generation). + pub fn signer(&self) -> &JwtSigner { + &self.signer + } +} + +/// Errors produced by this crate. +#[derive(Debug, thiserror::Error)] +pub enum OidcProviderError { + #[error("RSA key error: {0}")] + KeyError(String), + + #[error("JWT signing error: {0}")] + SigningError(String), + + #[error("credential exchange failed: {0}")] + ExchangeError(String), + + #[error("HTTP error: {0}")] + HttpError(String), +} + +impl From for s3_proxy_core::error::ProxyError { + fn from(e: OidcProviderError) -> Self { + s3_proxy_core::error::ProxyError::Internal(e.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{Duration, Utc}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Mock HTTP client that records calls and returns a preset AWS STS response. + #[derive(Clone)] + struct MockHttp { + call_count: Arc, + } + + impl MockHttp { + fn new() -> Self { + Self { + call_count: Arc::new(AtomicUsize::new(0)), + } + } + + fn calls(&self) -> usize { + self.call_count.load(Ordering::SeqCst) + } + } + + impl HttpExchange for MockHttp { + async fn post_form( + &self, + _url: &str, + _form: &[(&str, &str)], + ) -> Result { + self.call_count.fetch_add(1, Ordering::SeqCst); + let exp = (Utc::now() + Duration::hours(1)).to_rfc3339(); + Ok(format!( + r#" + + + AKID_MOCK + secret_mock + token_mock + {exp} + + + "# + )) + } + } + + fn test_signer() -> JwtSigner { + use rsa::pkcs8::EncodePrivateKey; + let mut rng = rand::rngs::OsRng; + let key = rsa::RsaPrivateKey::new(&mut rng, 2048).unwrap(); + let pem = key.to_pkcs8_pem(rsa::pkcs8::LineEnding::LF).unwrap(); + JwtSigner::from_pem(&pem, "test-kid".into(), 300).unwrap() + } + + #[tokio::test] + async fn get_credentials_returns_fresh_on_first_call() { + let http = MockHttp::new(); + let provider = OidcCredentialProvider::new( + test_signer(), + http.clone(), + "https://issuer.example.com".into(), + "sts.amazonaws.com".into(), + ); + + let exchange = exchange::aws::AwsExchange::new("arn:aws:iam::123:role/Test".into()); + let creds = provider + .get_credentials("role-a", &exchange, "my-sub", &[]) + .await + .unwrap(); + + assert_eq!(creds.access_key_id, "AKID_MOCK"); + assert_eq!(http.calls(), 1); + } + + #[tokio::test] + async fn get_credentials_uses_cache_on_second_call() { + let http = MockHttp::new(); + let provider = OidcCredentialProvider::new( + test_signer(), + http.clone(), + "https://issuer.example.com".into(), + "sts.amazonaws.com".into(), + ); + + let exchange = exchange::aws::AwsExchange::new("arn:aws:iam::123:role/Test".into()); + + // First call — hits mock HTTP + let creds1 = provider + .get_credentials("role-a", &exchange, "sub", &[]) + .await + .unwrap(); + assert_eq!(http.calls(), 1); + + // Second call — should use cache, no additional HTTP call + let creds2 = provider + .get_credentials("role-a", &exchange, "sub", &[]) + .await + .unwrap(); + assert_eq!(http.calls(), 1); + assert_eq!(creds1.access_key_id, creds2.access_key_id); + } + + #[tokio::test] + async fn different_cache_keys_make_separate_calls() { + let http = MockHttp::new(); + let provider = OidcCredentialProvider::new( + test_signer(), + http.clone(), + "https://issuer.example.com".into(), + "sts.amazonaws.com".into(), + ); + + let exchange = exchange::aws::AwsExchange::new("arn:aws:iam::123:role/Test".into()); + + provider + .get_credentials("role-a", &exchange, "sub", &[]) + .await + .unwrap(); + provider + .get_credentials("role-b", &exchange, "sub", &[]) + .await + .unwrap(); + + assert_eq!(http.calls(), 2); + } + + #[test] + fn signed_jwt_is_verifiable_via_jwks_public_key() { + use base64::Engine; + use rsa::pkcs1v15::VerifyingKey; + use rsa::signature::Verifier; + use rsa::{BigUint, RsaPublicKey}; + + let signer = test_signer(); + + // Sign a JWT + let token = signer.sign("sub", "iss", "aud", &[]).unwrap(); + + // Generate JWKS from the same signer + let jwks_str = jwks::jwks_json(signer.public_key(), signer.kid()); + let jwks: serde_json::Value = serde_json::from_str(&jwks_str).unwrap(); + + // Extract public key from JWKS + let key = &jwks["keys"][0]; + let b64 = &base64::engine::general_purpose::URL_SAFE_NO_PAD; + let n = BigUint::from_bytes_be(&b64.decode(key["n"].as_str().unwrap()).unwrap()); + let e = BigUint::from_bytes_be(&b64.decode(key["e"].as_str().unwrap()).unwrap()); + let reconstructed_key = RsaPublicKey::new(n, e).unwrap(); + + // Verify signature using the JWKS-derived key + let parts: Vec<&str> = token.split('.').collect(); + let signing_input = format!("{}.{}", parts[0], parts[1]); + let sig_bytes = b64.decode(parts[2]).unwrap(); + let signature = rsa::pkcs1v15::Signature::try_from(sig_bytes.as_slice()).unwrap(); + + let verifying_key = VerifyingKey::::new(reconstructed_key); + verifying_key + .verify(signing_input.as_bytes(), &signature) + .expect("JWT signed by JwtSigner should be verifiable via JWKS public key"); + } + + #[test] + fn error_converts_to_proxy_error() { + let err = OidcProviderError::ExchangeError("test".into()); + let proxy_err: s3_proxy_core::error::ProxyError = err.into(); + assert!(proxy_err.to_string().contains("test")); + assert_eq!(proxy_err.status_code(), 500); + } +} From a177971461495de0987c6563377969a60a801365 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 17:22:38 -0800 Subject: [PATCH 11/82] refactor: make key prefix configurable --- crates/libs/auth/src/lib.rs | 28 +++++++++++++++++++--------- crates/libs/auth/src/sts.rs | 3 ++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/crates/libs/auth/src/lib.rs b/crates/libs/auth/src/lib.rs index f16c664..25e9b5c 100644 --- a/crates/libs/auth/src/lib.rs +++ b/crates/libs/auth/src/lib.rs @@ -52,6 +52,7 @@ pub async fn assume_role_with_web_identity( role_arn: &str, web_identity_token: &str, duration_seconds: Option, + key_prefix: &str, ) -> Result { // Look up the role let role = config @@ -86,10 +87,7 @@ pub async fn assume_role_with_web_identity( let claims = jwks::verify_token(web_identity_token, key, issuer, &role)?; // Check subject conditions - let subject = claims - .get("sub") - .and_then(|v| v.as_str()) - .unwrap_or(""); + let subject = claims.get("sub").and_then(|v| v.as_str()).unwrap_or(""); if !role.subject_conditions.is_empty() { let matches = role @@ -109,7 +107,7 @@ pub async fn assume_role_with_web_identity( .unwrap_or(3600) .min(role.max_session_duration_secs); - let creds = sts::mint_temporary_credentials(&role, subject, duration); + let creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix); // Store them config.store_temporary_credential(&creds).await?; @@ -165,10 +163,22 @@ mod tests { #[test] fn test_subject_matching() { - assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/repo:*")); + assert!(subject_matches( + "repo:org/repo:ref:refs/heads/main", + "repo:org/repo:*" + )); assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "*")); - assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/repo:ref:refs/heads/main")); - assert!(!subject_matches("repo:org/repo:ref:refs/heads/main", "repo:other/*")); - assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "repo:org/*:ref:refs/heads/*")); + assert!(subject_matches( + "repo:org/repo:ref:refs/heads/main", + "repo:org/repo:ref:refs/heads/main" + )); + assert!(!subject_matches( + "repo:org/repo:ref:refs/heads/main", + "repo:other/*" + )); + assert!(subject_matches( + "repo:org/repo:ref:refs/heads/main", + "repo:org/*:ref:refs/heads/*" + )); } } diff --git a/crates/libs/auth/src/sts.rs b/crates/libs/auth/src/sts.rs index 249f465..70f975b 100644 --- a/crates/libs/auth/src/sts.rs +++ b/crates/libs/auth/src/sts.rs @@ -9,8 +9,9 @@ pub fn mint_temporary_credentials( role: &RoleConfig, source_identity: &str, duration_seconds: u64, + key_prefix: &str, ) -> TemporaryCredentials { - let access_key_id = format!("ASIA{}", generate_random_id(16)); + let access_key_id = format!("{}{}", key_prefix, generate_random_id(16)); let secret_access_key = generate_random_id(40); let session_token = generate_session_token(); From 66a007aa0b360db004f5550fa2f3679bcff72c34 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 19:31:08 -0800 Subject: [PATCH 12/82] refactor: auth -> sts --- CLAUDE.md | 2 +- Cargo.lock | 42 ++++++----- Cargo.toml | 6 +- README.md | 2 +- crates/libs/core/src/auth.rs | 24 +++--- crates/libs/core/src/resolver.rs | 8 -- crates/libs/core/src/s3/request.rs | 39 ---------- crates/libs/core/src/s3/response.rs | 52 ++----------- crates/libs/core/src/types.rs | 26 +------ crates/libs/{auth => sts}/Cargo.toml | 4 +- crates/libs/{auth => sts}/README.md | 25 ++++--- crates/libs/{auth => sts}/src/jwks.rs | 0 crates/libs/{auth => sts}/src/lib.rs | 18 +++-- crates/libs/sts/src/request.rs | 100 +++++++++++++++++++++++++ crates/libs/sts/src/responses.rs | 49 ++++++++++++ crates/libs/{auth => sts}/src/sts.rs | 0 crates/runtimes/cf-workers/Cargo.toml | 2 +- crates/runtimes/cf-workers/src/body.rs | 11 +-- crates/runtimes/cf-workers/src/lib.rs | 24 ++++-- crates/runtimes/server/Cargo.toml | 2 +- 20 files changed, 251 insertions(+), 185 deletions(-) rename crates/libs/{auth => sts}/Cargo.toml (86%) rename crates/libs/{auth => sts}/README.md (74%) rename crates/libs/{auth => sts}/src/jwks.rs (100%) rename crates/libs/{auth => sts}/src/lib.rs (92%) create mode 100644 crates/libs/sts/src/request.rs create mode 100644 crates/libs/sts/src/responses.rs rename crates/libs/{auth => sts}/src/sts.rs (100%) diff --git a/CLAUDE.md b/CLAUDE.md index 092aa1e..718402c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ The intention of this codebase is to serve as a data proxy for the Source Cooper ## Workspace Structure - `crates/libs/core` — Core proxy logic, traits, config, S3 request parsing -- `crates/libs/auth` — Authentication (SigV4 verification, JWT) +- `crates/libs/sts` — OIDC/STS token exchange (AssumeRoleWithWebIdentity, JWT validation) - `crates/runtimes/server` — Tokio/Hyper server runtime - `crates/runtimes/cf-workers` — Cloudflare Workers runtime (WASM) diff --git a/Cargo.lock b/Cargo.lock index 6333664..b1832ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2078,24 +2078,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "s3-proxy-auth" -version = "0.1.0" -dependencies = [ - "async-trait", - "base64", - "chrono", - "reqwest", - "rsa", - "s3-proxy-core", - "serde", - "serde_json", - "sha2", - "thiserror", - "tracing", - "uuid", -] - [[package]] name = "s3-proxy-cf-workers" version = "0.1.0" @@ -2113,9 +2095,9 @@ dependencies = [ "js-sys", "object_store", "quick-xml 0.37.5", - "s3-proxy-auth", "s3-proxy-core", "s3-proxy-source-coop", + "s3-proxy-sts", "serde", "serde_json", "thiserror", @@ -2186,9 +2168,9 @@ dependencies = [ "hyper-util", "object_store", "reqwest", - "s3-proxy-auth", "s3-proxy-core", "s3-proxy-source-coop", + "s3-proxy-sts", "serde", "thiserror", "tokio", @@ -2210,6 +2192,26 @@ dependencies = [ "url", ] +[[package]] +name = "s3-proxy-sts" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "chrono", + "quick-xml 0.37.5", + "reqwest", + "rsa", + "s3-proxy-core", + "serde", + "serde_json", + "sha2", + "thiserror", + "tracing", + "url", + "uuid", +] + [[package]] name = "schannel" version = "0.1.28" diff --git a/Cargo.toml b/Cargo.toml index f83cb30..7567989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "crates/libs/core", - "crates/libs/auth", + "crates/libs/sts", "crates/libs/oidc-provider", "crates/libs/source-coop", "crates/runtimes/server", @@ -12,7 +12,7 @@ members = [ # separately via: cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown default-members = [ "crates/libs/core", - "crates/libs/auth", + "crates/libs/sts", "crates/libs/oidc-provider", "crates/libs/source-coop", "crates/runtimes/server", @@ -72,6 +72,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Internal crates s3-proxy-core = { path = "crates/libs/core" } -s3-proxy-auth = { path = "crates/libs/auth" } +s3-proxy-sts = { path = "crates/libs/sts" } s3-proxy-oidc-provider = { path = "crates/libs/oidc-provider" } s3-proxy-source-coop = { path = "crates/libs/source-coop" } diff --git a/README.md b/README.md index 0ea6f6e..03de276 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ flowchart crates/ ├── libs/ # Libraries — not directly runnable │ ├── core/ (s3-proxy-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config providers -│ └── auth/ (s3-proxy-auth) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) +│ └── sts/ (s3-proxy-sts) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) └── runtimes/ # Runnable targets — one per deployment platform ├── server/ (s3-proxy-server) # Tokio/Hyper for container deployments └── cf-workers/ (s3-proxy-cf-workers) # Cloudflare Workers for edge deployments diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index fd7131a..0292576 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -48,14 +48,19 @@ pub fn parse_sigv4_auth(auth_header: &str) -> Result { } } - let credential = credential.ok_or_else(|| ProxyError::InvalidRequest("missing Credential".into()))?; - let signed_headers = signed_headers.ok_or_else(|| ProxyError::InvalidRequest("missing SignedHeaders".into()))?; - let signature = signature.ok_or_else(|| ProxyError::InvalidRequest("missing Signature".into()))?; + let credential = + credential.ok_or_else(|| ProxyError::InvalidRequest("missing Credential".into()))?; + let signed_headers = + signed_headers.ok_or_else(|| ProxyError::InvalidRequest("missing SignedHeaders".into()))?; + let signature = + signature.ok_or_else(|| ProxyError::InvalidRequest("missing Signature".into()))?; // Parse credential: AKID/date/region/service/aws4_request let cred_parts: Vec<&str> = credential.split('/').collect(); if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { - return Err(ProxyError::InvalidRequest("malformed credential scope".into())); + return Err(ProxyError::InvalidRequest( + "malformed credential scope".into(), + )); } Ok(SigV4Auth { @@ -210,7 +215,10 @@ pub fn authorize( if bucket_config.anonymous_access { // Anonymous users can only read let action = operation.action(); - if matches!(action, Action::GetObject | Action::HeadObject | Action::ListBucket) { + if matches!( + action, + Action::GetObject | Action::HeadObject | Action::ListBucket + ) { return Ok(()); } } @@ -252,10 +260,7 @@ pub fn authorize( if scope.prefixes.is_empty() { return true; // Full bucket access } - scope - .prefixes - .iter() - .any(|prefix| key.starts_with(prefix)) + scope.prefixes.iter().any(|prefix| key.starts_with(prefix)) }); if authorized { @@ -264,4 +269,3 @@ pub fn authorize( Err(ProxyError::AccessDenied) } } - diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index 06d162d..803f21a 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -91,14 +91,6 @@ impl RequestResolver for DefaultResolver

{ let operation = request::parse_s3_request(method, path, query, headers, host_style)?; tracing::debug!(operation = ?operation, "parsed S3 operation"); - // Handle STS requests separately (no bucket lookup needed) - if let S3Operation::AssumeRoleWithWebIdentity { .. } = &operation { - tracing::info!("STS AssumeRoleWithWebIdentity request"); - return Err(ProxyError::InvalidRequest( - "STS endpoint: use s3-proxy-auth crate for OIDC token exchange".into(), - )); - } - // Handle ListBuckets — returns virtual bucket list from config, no backend call if matches!(operation, S3Operation::ListBuckets) { let buckets = self.config.list_buckets().await?; diff --git a/crates/libs/core/src/s3/request.rs b/crates/libs/core/src/s3/request.rs index d696de1..731f563 100644 --- a/crates/libs/core/src/s3/request.rs +++ b/crates/libs/core/src/s3/request.rs @@ -15,20 +15,6 @@ pub fn parse_s3_request( _headers: &http::HeaderMap, host_style: HostStyle, ) -> Result { - // Check for STS actions in query params - if let Some(q) = query { - let params: Vec<(String, String)> = url::form_urlencoded::parse(q.as_bytes()) - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - - let action = params.iter().find(|(k, _)| k == "Action"); - if let Some((_, action_value)) = action { - if action_value == "AssumeRoleWithWebIdentity" { - return parse_sts_request(¶ms); - } - } - } - // GET / with path-style → ListBuckets (no bucket in path) if matches!(host_style, HostStyle::Path) && uri_path.trim_start_matches('/').is_empty() { if *method == Method::GET { @@ -164,28 +150,3 @@ fn parse_query_params(query: Option<&str>) -> Vec<(String, String)> { }) .unwrap_or_default() } - -fn parse_sts_request(params: &[(String, String)]) -> Result { - let role_arn = params - .iter() - .find(|(k, _)| k == "RoleArn") - .map(|(_, v)| v.clone()) - .ok_or_else(|| ProxyError::InvalidRequest("missing RoleArn".into()))?; - - let web_identity_token = params - .iter() - .find(|(k, _)| k == "WebIdentityToken") - .map(|(_, v)| v.clone()) - .ok_or_else(|| ProxyError::InvalidRequest("missing WebIdentityToken".into()))?; - - let duration_seconds = params - .iter() - .find(|(k, _)| k == "DurationSeconds") - .and_then(|(_, v)| v.parse().ok()); - - Ok(S3Operation::AssumeRoleWithWebIdentity { - role_arn, - web_identity_token, - duration_seconds, - }) -} diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs index 0e91db8..52608eb 100644 --- a/crates/libs/core/src/s3/response.rs +++ b/crates/libs/core/src/s3/response.rs @@ -32,7 +32,8 @@ impl ErrorResponse { pub fn to_xml(&self) -> String { format!( "\n{}", - xml_to_string(self).unwrap_or_else(|_| "InternalError".to_string()) + xml_to_string(self) + .unwrap_or_else(|_| "InternalError".to_string()) ) } } @@ -191,51 +192,6 @@ impl ListBucketResult { } } -/// STS AssumeRoleWithWebIdentity response. -#[derive(Debug, Serialize)] -#[serde(rename = "AssumeRoleWithWebIdentityResponse")] -pub struct AssumeRoleWithWebIdentityResponse { - #[serde(rename = "AssumeRoleWithWebIdentityResult")] - pub result: AssumeRoleWithWebIdentityResult, -} - -#[derive(Debug, Serialize)] -pub struct AssumeRoleWithWebIdentityResult { - #[serde(rename = "Credentials")] - pub credentials: StsCredentials, - #[serde(rename = "AssumedRoleUser")] - pub assumed_role_user: AssumedRoleUser, -} - -#[derive(Debug, Serialize)] -pub struct StsCredentials { - #[serde(rename = "AccessKeyId")] - pub access_key_id: String, - #[serde(rename = "SecretAccessKey")] - pub secret_access_key: String, - #[serde(rename = "SessionToken")] - pub session_token: String, - #[serde(rename = "Expiration")] - pub expiration: String, -} - -#[derive(Debug, Serialize)] -pub struct AssumedRoleUser { - #[serde(rename = "AssumedRoleId")] - pub assumed_role_id: String, - #[serde(rename = "Arn")] - pub arn: String, -} - -impl AssumeRoleWithWebIdentityResponse { - pub fn to_xml(&self) -> String { - format!( - "\n{}", - xml_to_string(self).unwrap_or_default() - ) - } -} - #[cfg(test)] mod tests { use super::*; @@ -264,7 +220,9 @@ mod tests { let xml = result.to_xml(); assert!(xml.starts_with("")); - assert!(xml.contains("")); + assert!( + xml.contains("") + ); assert!(xml.contains("my-bucket")); assert!(xml.contains("photos/image.jpg")); assert!(xml.contains("1024")); diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index db4827e..ac71c91 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -142,12 +142,8 @@ pub struct TemporaryCredentials { #[derive(Debug, Clone)] pub enum ResolvedIdentity { Anonymous, - LongLived { - credential: StoredCredential, - }, - Temporary { - credentials: TemporaryCredentials, - }, + LongLived { credential: StoredCredential }, + Temporary { credentials: TemporaryCredentials }, } /// The parsed S3 operation extracted from an incoming request. @@ -198,12 +194,6 @@ pub enum S3Operation { }, /// List all virtual buckets exposed by the proxy. ListBuckets, - /// STS AssumeRoleWithWebIdentity (served on the same endpoint). - AssumeRoleWithWebIdentity { - role_arn: String, - web_identity_token: String, - duration_seconds: Option, - }, } impl S3Operation { @@ -220,7 +210,6 @@ impl S3Operation { S3Operation::AbortMultipartUpload { .. } => Action::AbortMultipartUpload, S3Operation::DeleteObject { .. } => Action::DeleteObject, S3Operation::ListBuckets => Action::ListBucket, - S3Operation::AssumeRoleWithWebIdentity { .. } => Action::GetObject, // STS is handled separately } } @@ -237,7 +226,6 @@ impl S3Operation { | S3Operation::AbortMultipartUpload { bucket, .. } | S3Operation::DeleteObject { bucket, .. } => Some(bucket), S3Operation::ListBuckets => None, - S3Operation::AssumeRoleWithWebIdentity { .. } => None, } } @@ -253,8 +241,7 @@ impl S3Operation { | S3Operation::AbortMultipartUpload { key, .. } | S3Operation::DeleteObject { key, .. } => key, S3Operation::ListBucket { .. } - | S3Operation::ListBuckets - | S3Operation::AssumeRoleWithWebIdentity { .. } => "", + | S3Operation::ListBuckets => "", } } } @@ -301,13 +288,6 @@ mod tests { assert_eq!(op.bucket(), Some("my-bucket")); assert_eq!(S3Operation::ListBuckets.bucket(), None); - - let op = S3Operation::AssumeRoleWithWebIdentity { - role_arn: "arn".into(), - web_identity_token: "tok".into(), - duration_seconds: None, - }; - assert_eq!(op.bucket(), None); } #[test] diff --git a/crates/libs/auth/Cargo.toml b/crates/libs/sts/Cargo.toml similarity index 86% rename from crates/libs/auth/Cargo.toml rename to crates/libs/sts/Cargo.toml index 73ac2a2..02c3593 100644 --- a/crates/libs/auth/Cargo.toml +++ b/crates/libs/sts/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-auth" +name = "s3-proxy-sts" version.workspace = true edition.workspace = true license.workspace = true @@ -18,3 +18,5 @@ rsa.workspace = true sha2.workspace = true reqwest.workspace = true tracing.workspace = true +quick-xml.workspace = true +url.workspace = true diff --git a/crates/libs/auth/README.md b/crates/libs/sts/README.md similarity index 74% rename from crates/libs/auth/README.md rename to crates/libs/sts/README.md index 416e7cf..19ac2ca 100644 --- a/crates/libs/auth/README.md +++ b/crates/libs/sts/README.md @@ -1,4 +1,4 @@ -# s3-proxy-auth +# s3-proxy-sts OIDC token exchange and STS credential minting for the S3 proxy gateway. Implements the `AssumeRoleWithWebIdentity` flow, allowing workloads like GitHub Actions to exchange OIDC JWTs for temporary, scoped S3 credentials. @@ -10,7 +10,7 @@ GitHub Actions (or any OIDC provider) │ JWT (signed by provider) ▼ ┌─────────────────────────────┐ -│ s3-proxy-auth │ +│ s3-proxy-sts │ │ │ │ 1. Decode JWT header │ │ 2. Fetch JWKS from issuer │ @@ -41,9 +41,11 @@ If you need to use a different HTTP client for JWKS fetching (e.g., the Workers ``` src/ -├── lib.rs Entry point: assume_role_with_web_identity(), subject glob matching -├── jwks.rs JWKS fetching, JWK parsing, JWT signature verification -└── sts.rs Temporary credential minting (AccessKeyId/SecretAccessKey/SessionToken) +├── lib.rs Entry point: assume_role_with_web_identity(), subject glob matching +├── request.rs STS request parsing (AssumeRoleWithWebIdentity query params) +├── responses.rs STS XML response serialization +├── jwks.rs JWKS fetching, JWK parsing, JWT signature verification +└── sts.rs Temporary credential minting (AccessKeyId/SecretAccessKey/SessionToken) ``` ## Usage @@ -51,13 +53,18 @@ src/ Called by the proxy handler when it receives an STS `AssumeRoleWithWebIdentity` request: ```rust -use s3_proxy_auth::assume_role_with_web_identity; +use s3_proxy_sts::assume_role_with_web_identity; +use s3_proxy_sts::request::{StsRequest, try_parse_sts_request}; + +// Parse from query string +let sts_request = try_parse_sts_request(Some(query)) + .transpose()? // Option> → Result> + .expect("STS request"); let creds = assume_role_with_web_identity( &config_provider, - "github-actions-deployer", // role ARN - &jwt_token, // OIDC token from the client - Some(3600), // session duration (seconds) + &sts_request, + "TEMPKEY", // key prefix for minted credentials ).await?; // creds.access_key_id, creds.secret_access_key, creds.session_token diff --git a/crates/libs/auth/src/jwks.rs b/crates/libs/sts/src/jwks.rs similarity index 100% rename from crates/libs/auth/src/jwks.rs rename to crates/libs/sts/src/jwks.rs diff --git a/crates/libs/auth/src/lib.rs b/crates/libs/sts/src/lib.rs similarity index 92% rename from crates/libs/auth/src/lib.rs rename to crates/libs/sts/src/lib.rs index 25e9b5c..00415cd 100644 --- a/crates/libs/auth/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -16,9 +16,12 @@ //! The client then uses these credentials to sign S3 requests normally. pub mod jwks; +pub mod request; +pub mod responses; pub mod sts; use base64::Engine; +use request::StsRequest; use s3_proxy_core::config::ConfigProvider; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::TemporaryCredentials; @@ -49,19 +52,17 @@ fn jwt_decode_unverified( /// Validate an OIDC token and mint temporary credentials. pub async fn assume_role_with_web_identity( config: &C, - role_arn: &str, - web_identity_token: &str, - duration_seconds: Option, + sts_request: &StsRequest, key_prefix: &str, ) -> Result { // Look up the role let role = config - .get_role(role_arn) + .get_role(&sts_request.role_arn) .await? - .ok_or_else(|| ProxyError::RoleNotFound(role_arn.to_string()))?; + .ok_or_else(|| ProxyError::RoleNotFound(sts_request.role_arn.to_string()))?; // Decode the JWT header and claims without verification to extract issuer and kid - let (header, insecure_claims) = jwt_decode_unverified(web_identity_token)?; + let (header, insecure_claims) = jwt_decode_unverified(&sts_request.web_identity_token)?; let issuer = insecure_claims .get("iss") @@ -84,7 +85,7 @@ pub async fn assume_role_with_web_identity( .ok_or_else(|| ProxyError::InvalidOidcToken("JWT missing kid".into()))?; let key = jwks::find_key(&jwks, kid)?; - let claims = jwks::verify_token(web_identity_token, key, issuer, &role)?; + let claims = jwks::verify_token(&sts_request.web_identity_token, key, issuer, &role)?; // Check subject conditions let subject = claims.get("sub").and_then(|v| v.as_str()).unwrap_or(""); @@ -103,7 +104,8 @@ pub async fn assume_role_with_web_identity( } // Mint temporary credentials - let duration = duration_seconds + let duration = sts_request + .duration_seconds .unwrap_or(3600) .min(role.max_session_duration_secs); diff --git a/crates/libs/sts/src/request.rs b/crates/libs/sts/src/request.rs new file mode 100644 index 0000000..d93e864 --- /dev/null +++ b/crates/libs/sts/src/request.rs @@ -0,0 +1,100 @@ +//! STS request parsing. +//! +//! Extracts `AssumeRoleWithWebIdentity` parameters from query strings. + +use s3_proxy_core::error::ProxyError; + +/// Parsed STS `AssumeRoleWithWebIdentity` request parameters. +#[derive(Debug, Clone)] +pub struct StsRequest { + pub role_arn: String, + pub web_identity_token: String, + pub duration_seconds: Option, +} + +/// Try to parse an STS request from the query string. +/// +/// Returns `None` if the query does not contain `Action=AssumeRoleWithWebIdentity` +/// (i.e., this is not an STS request). Returns `Some(Ok(..))` on success or +/// `Some(Err(..))` if it is an STS request but required parameters are missing. +pub fn try_parse_sts_request(query: Option<&str>) -> Option> { + let q = query?; + let params: Vec<(String, String)> = url::form_urlencoded::parse(q.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let action = params.iter().find(|(k, _)| k == "Action"); + match action { + Some((_, value)) if value == "AssumeRoleWithWebIdentity" => {} + _ => return None, + } + + Some(parse_sts_params(¶ms)) +} + +fn parse_sts_params(params: &[(String, String)]) -> Result { + let role_arn = params + .iter() + .find(|(k, _)| k == "RoleArn") + .map(|(_, v)| v.clone()) + .ok_or_else(|| ProxyError::InvalidRequest("missing RoleArn".into()))?; + + let web_identity_token = params + .iter() + .find(|(k, _)| k == "WebIdentityToken") + .map(|(_, v)| v.clone()) + .ok_or_else(|| ProxyError::InvalidRequest("missing WebIdentityToken".into()))?; + + let duration_seconds = params + .iter() + .find(|(k, _)| k == "DurationSeconds") + .and_then(|(_, v)| v.parse().ok()); + + Ok(StsRequest { + role_arn, + web_identity_token, + duration_seconds, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_not_sts_request() { + assert!(try_parse_sts_request(None).is_none()); + assert!(try_parse_sts_request(Some("prefix=foo/")).is_none()); + assert!(try_parse_sts_request(Some("Action=ListBuckets")).is_none()); + } + + #[test] + fn test_valid_sts_request() { + let query = "Action=AssumeRoleWithWebIdentity&RoleArn=my-role&WebIdentityToken=tok123"; + let result = try_parse_sts_request(Some(query)).unwrap().unwrap(); + assert_eq!(result.role_arn, "my-role"); + assert_eq!(result.web_identity_token, "tok123"); + assert_eq!(result.duration_seconds, None); + } + + #[test] + fn test_sts_request_with_duration() { + let query = "Action=AssumeRoleWithWebIdentity&RoleArn=r&WebIdentityToken=t&DurationSeconds=7200"; + let result = try_parse_sts_request(Some(query)).unwrap().unwrap(); + assert_eq!(result.duration_seconds, Some(7200)); + } + + #[test] + fn test_missing_role_arn() { + let query = "Action=AssumeRoleWithWebIdentity&WebIdentityToken=tok"; + let result = try_parse_sts_request(Some(query)).unwrap(); + assert!(result.is_err()); + } + + #[test] + fn test_missing_web_identity_token() { + let query = "Action=AssumeRoleWithWebIdentity&RoleArn=role"; + let result = try_parse_sts_request(Some(query)).unwrap(); + assert!(result.is_err()); + } +} diff --git a/crates/libs/sts/src/responses.rs b/crates/libs/sts/src/responses.rs new file mode 100644 index 0000000..601dffa --- /dev/null +++ b/crates/libs/sts/src/responses.rs @@ -0,0 +1,49 @@ +//! STS XML response serialization. + +use quick_xml::se::to_string as xml_to_string; +use serde::Serialize; + +/// STS AssumeRoleWithWebIdentity response. +#[derive(Debug, Serialize)] +#[serde(rename = "AssumeRoleWithWebIdentityResponse")] +pub struct AssumeRoleWithWebIdentityResponse { + #[serde(rename = "AssumeRoleWithWebIdentityResult")] + pub result: AssumeRoleWithWebIdentityResult, +} + +#[derive(Debug, Serialize)] +pub struct AssumeRoleWithWebIdentityResult { + #[serde(rename = "Credentials")] + pub credentials: StsCredentials, + #[serde(rename = "AssumedRoleUser")] + pub assumed_role_user: AssumedRoleUser, +} + +#[derive(Debug, Serialize)] +pub struct StsCredentials { + #[serde(rename = "AccessKeyId")] + pub access_key_id: String, + #[serde(rename = "SecretAccessKey")] + pub secret_access_key: String, + #[serde(rename = "SessionToken")] + pub session_token: String, + #[serde(rename = "Expiration")] + pub expiration: String, +} + +#[derive(Debug, Serialize)] +pub struct AssumedRoleUser { + #[serde(rename = "AssumedRoleId")] + pub assumed_role_id: String, + #[serde(rename = "Arn")] + pub arn: String, +} + +impl AssumeRoleWithWebIdentityResponse { + pub fn to_xml(&self) -> String { + format!( + "\n{}", + xml_to_string(self).unwrap_or_default() + ) + } +} diff --git a/crates/libs/auth/src/sts.rs b/crates/libs/sts/src/sts.rs similarity index 100% rename from crates/libs/auth/src/sts.rs rename to crates/libs/sts/src/sts.rs diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index b945285..3571c5d 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] s3-proxy-core = { workspace = true, features = [] } -s3-proxy-auth.workspace = true +s3-proxy-sts.workspace = true s3-proxy-source-coop.workspace = true bytes.workspace = true http.workspace = true diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs index 8d4fb42..370fa1c 100644 --- a/crates/runtimes/cf-workers/src/body.rs +++ b/crates/runtimes/cf-workers/src/body.rs @@ -14,9 +14,7 @@ use worker::{Headers, Response}; /// Stream bodies are bridged to JS `ReadableStream` via a `TransformStream`: /// a spawn_local task reads Rust stream chunks and writes them to the /// writable side; the readable side is used for the Response. -pub fn build_worker_response( - result: ProxyResult, -) -> Result { +pub fn build_worker_response(result: ProxyResult) -> Result { let resp_headers = Headers::new(); for (key, value) in result.headers.iter() { if let Ok(v) = value.to_str() { @@ -25,6 +23,7 @@ pub fn build_worker_response( } match result.body { + // TODO: This seems like it violates the need to keep streams in the JS form ProxyResponseBody::Stream(stream) => { // Bridge Rust Stream -> JS ReadableStream via TransformStream let transform = web_sys::TransformStream::new() @@ -42,8 +41,10 @@ pub fn build_worker_response( match chunk_result { Ok(bytes) => { let uint8 = Uint8Array::from(bytes.as_ref()); - if let Err(_) = - wasm_bindgen_futures::JsFuture::from(writer.write_with_chunk(&uint8.into())).await + if let Err(_) = wasm_bindgen_futures::JsFuture::from( + writer.write_with_chunk(&uint8.into()), + ) + .await { break; } diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 2c4202d..a0814b4 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -102,9 +102,12 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { } } - let http_client = client::WorkerHttpClient; - let api_client = - SourceApiClient::new(http_client, source_api_url.to_string(), source_api_key, cache_ttls); + let api_client = SourceApiClient::new( + client::WorkerHttpClient, + source_api_url.to_string(), + source_api_key, + cache_ttls, + ); let resolver = SourceCoopResolver::new(api_client); let handler = ProxyHandler::new(client::WorkerBackend, resolver); @@ -121,7 +124,10 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { // - JS object (e.g., set via `[vars.PROXY_CONFIG]` table in wrangler.toml) let config = if let Ok(var) = env.var("PROXY_CONFIG") { let config_str = var.to_string(); - tracing::debug!(config_len = config_str.len(), "loaded PROXY_CONFIG as string"); + tracing::debug!( + config_len = config_str.len(), + "loaded PROXY_CONFIG as string" + ); StaticProvider::from_json(&config_str) .map_err(|e| worker::Error::RustError(format!("config error: {}", e)))? } else { @@ -154,13 +160,15 @@ async fn read_request_body(req: &Request) -> Result { let response = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) .map_err(|e| worker::Error::RustError(format!("failed to wrap stream: {:?}", e)))?; - let array_buffer_promise = response - .array_buffer() - .map_err(|e| worker::Error::RustError(format!("failed to get arrayBuffer: {:?}", e)))?; + let array_buffer_promise = response.array_buffer().map_err(|e| { + worker::Error::RustError(format!("failed to get arrayBuffer: {:?}", e)) + })?; let array_buffer = wasm_bindgen_futures::JsFuture::from(array_buffer_promise) .await - .map_err(|e| worker::Error::RustError(format!("failed to read arrayBuffer: {:?}", e)))?; + .map_err(|e| { + worker::Error::RustError(format!("failed to read arrayBuffer: {:?}", e)) + })?; let uint8 = js_sys::Uint8Array::new(&array_buffer); Ok(bytes::Bytes::from(uint8.to_vec())) diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index 035551e..a7b12ca 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -7,7 +7,7 @@ description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] s3-proxy-core = { workspace = true, features = ["azure", "gcp"] } -s3-proxy-auth.workspace = true +s3-proxy-sts.workspace = true s3-proxy-source-coop.workspace = true tokio.workspace = true hyper.workspace = true From 17ac6e31aae0d030210bf279c54db5113a0bb0e3 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 19:31:39 -0800 Subject: [PATCH 13/82] docs: continue to buildout architecture --- ARCHITECTURE.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6fada57..cc53e92 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,5 +1,13 @@ # Source Data Proxy Architecture +## Data Proxy + +The core function of this system is to operate an S3-compliant API that proxies requests to appropriate object storage backends (e.g. MinIO, AWS S3, Cloudflare R2, Azure Blobstore). + +## Runtime + +The system is designed to operate in various runtime environments. Chiefly, these includes operating as a traditional server running on a Linux server or containerized environment (e.g. ECS, K8s), or running in WASM on Cloudflare Workers. + ## Authentication ### How clients authenticate with Source Data Proxy @@ -106,4 +114,8 @@ The proxy's OIDC discovery endpoints (`/.well-known/openid-configuration` and JW For object storage backends that are unable to utilize the Source Data Proxy as an Identity Provider, the Data Proxy also stores long-term access credentials provided by the administrators of the object storage backend. These credentials will be used to authenticate when the Data Proxy needs to interact with the object storage backend. [^vercel-oidc]: https://vercel.com/docs/oidc/aws -[^github-oidc]: https://docs.github.com/en/actions/concepts/security/openid-connect \ No newline at end of file +[^github-oidc]: https://docs.github.com/en/actions/concepts/security/openid-connect + +## Modularity + +The primary focus of this codebase is to serve as a data proxy for the [Source Cooperative](https://source.coop). However, it is built in a modular fashion to support reuse by others who have similar needs. \ No newline at end of file From 4e3cd048734aa29c0877de01cde1866283418236 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 21 Feb 2026 19:35:15 -0800 Subject: [PATCH 14/82] fix: add chromo feature --- Cargo.lock | 4 ++++ Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b1832ca..8c0a19b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2473,6 +2473,7 @@ checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64", "bytes", + "chrono", "crc", "crossbeam-queue", "either", @@ -2550,6 +2551,7 @@ dependencies = [ "bitflags", "byteorder", "bytes", + "chrono", "crc", "digest", "dotenvy", @@ -2591,6 +2593,7 @@ dependencies = [ "base64", "bitflags", "byteorder", + "chrono", "crc", "dotenvy", "etcetera", @@ -2625,6 +2628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", + "chrono", "flume", "futures-channel", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 7567989..d546e0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" # Config backends aws-sdk-dynamodb = "1" -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres"] } +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono"] } # HTTP body http-body = "1" From 0ba05a4c30d57ebf82c3153e57cf2196a5fb039a Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 22 Feb 2026 07:21:13 -0800 Subject: [PATCH 15/82] refactor: remove streaming body conversions --- crates/libs/core/src/backend.rs | 27 +++ crates/libs/core/src/proxy.rs | 248 ++++++++++++++++++----- crates/libs/core/src/response_body.rs | 20 +- crates/runtimes/cf-workers/src/body.rs | 35 +++- crates/runtimes/cf-workers/src/client.rs | 107 ++++++++-- crates/runtimes/cf-workers/src/lib.rs | 4 + crates/runtimes/server/src/body.rs | 17 +- crates/runtimes/server/src/client.rs | 40 +++- 8 files changed, 416 insertions(+), 82 deletions(-) diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index bd8faed..73c9403 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -33,6 +33,12 @@ use object_store::gcp::GoogleCloudStorageBuilder; /// - Server runtime: uses `reqwest` for raw HTTP, default `object_store` HTTP connector /// - Worker runtime: uses `web_sys::fetch` for raw HTTP, custom `FetchConnector` for `object_store` pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { + /// The runtime's native streaming body type. + /// + /// CF Workers uses `Option` to avoid JS/Rust + /// stream round-trips. The server runtime uses a reqwest byte stream. + type NativeBody: MaybeSend + 'static; + /// Create an `ObjectStore` for the given bucket configuration. fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; @@ -45,6 +51,17 @@ pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { headers: HeaderMap, body: Bytes, ) -> impl Future> + MaybeSend; + + /// Send a raw HTTP request and return the response with a streaming body. + /// + /// Used for GET/HEAD on S3 backends to avoid double stream conversion + /// (JS→Rust→JS on Workers, or unnecessary buffering on server). + fn send_streaming( + &self, + method: http::Method, + url: String, + headers: HeaderMap, + ) -> impl Future, ProxyError>> + MaybeSend; } /// Response from a raw HTTP request to a backend. @@ -54,6 +71,16 @@ pub struct RawResponse { pub body: Bytes, } +/// Response from a streaming HTTP request to a backend. +/// +/// The body is the runtime's native streaming type, avoiding +/// intermediate conversion through Rust stream types. +pub struct RawStreamingResponse { + pub status: u16, + pub headers: HeaderMap, + pub body: N, +} + /// Wrapper around provider-specific `object_store` builders. /// /// Runtimes use [`build_object_store`] and inject their HTTP connector via diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 6e4da8d..bc55d50 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -1,8 +1,9 @@ //! The main proxy handler that ties together resolution and backend forwarding. //! //! [`ProxyHandler`] is generic over the runtime's backend and request resolver. -//! GET/HEAD/PUT/LIST operations go through `object_store`; multipart operations -//! use raw signed HTTP requests. +//! GET/HEAD operations use `send_streaming` for S3 backends (avoiding double +//! stream conversion) and `object_store` for non-S3 backends. Multipart +//! operations use raw signed HTTP requests. use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLOAD}; use crate::error::ProxyError; @@ -50,7 +51,7 @@ where query: Option<&str>, headers: &HeaderMap, body: Bytes, - ) -> ProxyResult { + ) -> ProxyResult { let request_id = Uuid::new_v4().to_string(); tracing::info!( @@ -93,7 +94,7 @@ where query: Option<&str>, headers: &HeaderMap, body: Bytes, - ) -> Result { + ) -> Result, ProxyError> { let action = self.resolver.resolve(&method, path, query, headers).await?; match action { @@ -132,13 +133,13 @@ where original_headers: &HeaderMap, body: Bytes, list_rewrite: Option<&ListRewrite>, - ) -> Result { + ) -> Result, ProxyError> { match operation { S3Operation::GetObject { key, .. } => { self.handle_get(bucket_config, key, original_headers).await } S3Operation::HeadObject { key, .. } => { - self.handle_head(bucket_config, key).await + self.handle_head(bucket_config, key, original_headers).await } S3Operation::PutObject { key, .. } => { self.handle_put(bucket_config, key, body).await @@ -168,13 +169,77 @@ where } } - /// GET via object_store + /// GET — uses `send_streaming` for S3 backends (zero-copy native body), + /// falls back to `object_store` for non-S3 backends. async fn handle_get( &self, config: &BucketConfig, key: &str, headers: &HeaderMap, - ) -> Result { + ) -> Result, ProxyError> { + if config.supports_s3_multipart() { + return self.handle_get_s3(config, key, headers).await; + } + self.handle_get_object_store(config, key, headers).await + } + + /// GET for S3 backends via `send_streaming` — the response body stays in + /// the runtime's native type, avoiding double JS/Rust stream conversion. + async fn handle_get_s3( + &self, + config: &BucketConfig, + key: &str, + headers: &HeaderMap, + ) -> Result, ProxyError> { + let operation = S3Operation::GetObject { + bucket: String::new(), + key: key.to_string(), + }; + let backend_url = build_backend_url(config, &operation)?; + + let mut req_headers = HeaderMap::new(); + + // Forward conditional and range headers + for header_name in &[ + "range", + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + ] { + if let Some(val) = headers.get(*header_name) { + req_headers.insert(*header_name, val.clone()); + } + } + + sign_s3_request(&Method::GET, &backend_url, &mut req_headers, config, UNSIGNED_PAYLOAD)?; + + tracing::debug!(backend_url = %backend_url, "GET via send_streaming (S3)"); + + let raw = self.backend.send_streaming(Method::GET, backend_url, req_headers).await?; + + // Forward response headers from the backend + let mut resp_headers = HeaderMap::new(); + for header_name in STREAMING_RESPONSE_HEADERS { + if let Some(val) = raw.headers.get(*header_name) { + resp_headers.insert(*header_name, val.clone()); + } + } + + Ok(ProxyResult { + status: raw.status, + headers: resp_headers, + body: ProxyResponseBody::Native(raw.body), + }) + } + + /// GET for non-S3 backends via `object_store`. + async fn handle_get_object_store( + &self, + config: &BucketConfig, + key: &str, + headers: &HeaderMap, + ) -> Result, ProxyError> { let store = self.backend.create_store(config)?; let path = build_object_path(config, key); @@ -263,12 +328,74 @@ where }) } - /// HEAD via object_store + /// HEAD — uses `send_streaming` for S3 backends (richer headers from backend), + /// falls back to `object_store` for non-S3 backends. async fn handle_head( &self, config: &BucketConfig, key: &str, - ) -> Result { + headers: &HeaderMap, + ) -> Result, ProxyError> { + if config.supports_s3_multipart() { + return self.handle_head_s3(config, key, headers).await; + } + self.handle_head_object_store(config, key).await + } + + /// HEAD for S3 backends via `send_streaming`. + async fn handle_head_s3( + &self, + config: &BucketConfig, + key: &str, + headers: &HeaderMap, + ) -> Result, ProxyError> { + let operation = S3Operation::HeadObject { + bucket: String::new(), + key: key.to_string(), + }; + let backend_url = build_backend_url(config, &operation)?; + + let mut req_headers = HeaderMap::new(); + + // Forward conditional headers + for header_name in &[ + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + ] { + if let Some(val) = headers.get(*header_name) { + req_headers.insert(*header_name, val.clone()); + } + } + + sign_s3_request(&Method::HEAD, &backend_url, &mut req_headers, config, UNSIGNED_PAYLOAD)?; + + tracing::debug!(backend_url = %backend_url, "HEAD via send_streaming (S3)"); + + let raw = self.backend.send_streaming(Method::HEAD, backend_url, req_headers).await?; + + // Forward response headers from the backend + let mut resp_headers = HeaderMap::new(); + for header_name in STREAMING_RESPONSE_HEADERS { + if let Some(val) = raw.headers.get(*header_name) { + resp_headers.insert(*header_name, val.clone()); + } + } + + Ok(ProxyResult { + status: raw.status, + headers: resp_headers, + body: ProxyResponseBody::Empty, + }) + } + + /// HEAD for non-S3 backends via `object_store`. + async fn handle_head_object_store( + &self, + config: &BucketConfig, + key: &str, + ) -> Result, ProxyError> { let store = self.backend.create_store(config)?; let path = build_object_path(config, key); @@ -307,7 +434,7 @@ where config: &BucketConfig, key: &str, body: Bytes, - ) -> Result { + ) -> Result, ProxyError> { let store = self.backend.create_store(config)?; let path = build_object_path(config, key); @@ -336,7 +463,7 @@ where &self, config: &BucketConfig, key: &str, - ) -> Result { + ) -> Result, ProxyError> { let store = self.backend.create_store(config)?; let path = build_object_path(config, key); @@ -360,7 +487,7 @@ where config: &BucketConfig, raw_query: Option<&str>, list_rewrite: Option<&ListRewrite>, - ) -> Result { + ) -> Result, ProxyError> { let store = self.backend.create_store(config)?; // Extract prefix from query string @@ -430,7 +557,7 @@ where bucket_config: &BucketConfig, original_headers: &HeaderMap, body: Bytes, - ) -> Result { + ) -> Result, ProxyError> { let backend_url = build_backend_url(bucket_config, operation)?; tracing::debug!(backend_url = %backend_url, "multipart via raw HTTP"); @@ -448,39 +575,13 @@ where } } - // Sign the request if credentials are configured - let access_key = bucket_config.option("access_key_id").unwrap_or(""); - let secret_key = bucket_config.option("secret_access_key").unwrap_or(""); - let region = bucket_config.option("region").unwrap_or("us-east-1"); - let has_credentials = !access_key.is_empty() && !secret_key.is_empty(); - - let parsed_url = Url::parse(&backend_url) - .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; - let payload_hash = if body.is_empty() { UNSIGNED_PAYLOAD.to_string() } else { hash_payload(&body) }; - if has_credentials { - let signer = S3RequestSigner::new( - access_key.to_string(), - secret_key.to_string(), - region.to_string(), - ); - signer.sign_request(method, &parsed_url, &mut headers, &payload_hash)?; - } else { - let host = parsed_url - .host_str() - .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; - let host_header = if let Some(port) = parsed_url.port() { - format!("{}:{}", host, port) - } else { - host.to_string() - }; - headers.insert("host", host_header.parse().unwrap()); - } + sign_s3_request(method, &backend_url, &mut headers, bucket_config, &payload_hash)?; let raw_resp = self .backend @@ -497,14 +598,27 @@ where } } -/// The result of handling a proxy request. -pub struct ProxyResult { +/// The result of handling a proxy request, generic over the native body type. +pub struct ProxyResult { pub status: u16, pub headers: HeaderMap, - pub body: ProxyResponseBody, + pub body: ProxyResponseBody, } -fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { +/// Headers to forward from backend streaming responses. +const STREAMING_RESPONSE_HEADERS: &[&str] = &[ + "content-type", + "content-length", + "content-range", + "etag", + "last-modified", + "accept-ranges", + "content-encoding", + "content-disposition", + "cache-control", +]; + +fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { let xml = ErrorResponse::from_proxy_error(err, resource, request_id).to_xml(); let body = ProxyResponseBody::from_bytes(Bytes::from(xml)); let mut headers = HeaderMap::new(); @@ -517,6 +631,47 @@ fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyRe } } +/// Sign an outbound S3 request using credentials from the bucket config. +/// +/// If credentials are configured (`access_key_id` + `secret_access_key`), +/// applies SigV4 signing. Otherwise, just sets the Host header. +fn sign_s3_request( + method: &Method, + url: &str, + headers: &mut HeaderMap, + config: &BucketConfig, + payload_hash: &str, +) -> Result<(), ProxyError> { + let access_key = config.option("access_key_id").unwrap_or(""); + let secret_key = config.option("secret_access_key").unwrap_or(""); + let region = config.option("region").unwrap_or("us-east-1"); + let has_credentials = !access_key.is_empty() && !secret_key.is_empty(); + + let parsed_url = Url::parse(url) + .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; + + if has_credentials { + let signer = S3RequestSigner::new( + access_key.to_string(), + secret_key.to_string(), + region.to_string(), + ); + signer.sign_request(method, &parsed_url, headers, payload_hash)?; + } else { + let host = parsed_url + .host_str() + .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; + let host_header = if let Some(port) = parsed_url.port() { + format!("{}:{}", host, port) + } else { + host.to_string() + }; + headers.insert("host", host_header.parse().unwrap()); + } + + Ok(()) +} + /// Build an object_store Path from a bucket config and client-visible key. fn build_object_path(config: &BucketConfig, key: &str) -> object_store::path::Path { let mut full_key = String::new(); @@ -658,7 +813,8 @@ fn parse_range_header(value: &str) -> Option { } } -fn build_backend_url( +/// Build the backend URL for an S3 operation. +pub fn build_backend_url( config: &BucketConfig, operation: &S3Operation, ) -> Result { diff --git a/crates/libs/core/src/response_body.rs b/crates/libs/core/src/response_body.rs index 2eb3316..6aaea0d 100644 --- a/crates/libs/core/src/response_body.rs +++ b/crates/libs/core/src/response_body.rs @@ -1,26 +1,32 @@ //! Response body type for the proxy. //! -//! [`ProxyResponseBody`] replaces the old generic body type parameter. -//! All runtimes convert this to their native response type at the edge. +//! [`ProxyResponseBody`] is generic over a native body type `N` so that each +//! runtime can carry its platform-native streaming body through the handler +//! without intermediate conversion (e.g. JS ReadableStream on CF Workers). use bytes::Bytes; use futures::stream::BoxStream; /// The body of a proxy response. /// -/// This is no longer generic — all runtimes work with this concrete type -/// and convert to their native response format. -pub enum ProxyResponseBody { - /// Streaming response from `object_store` GET. +/// Generic over `N`, the runtime's native streaming body type. +/// The default `N = ()` is used for responses that don't carry a native body +/// (errors, list XML, HEAD, etc.). +pub enum ProxyResponseBody { + /// Streaming response from `object_store` GET (non-S3 backends). /// Bytes arrive lazily in chunks. Stream(BoxStream<'static, Result>), /// Fixed bytes (error XML, list XML, multipart XML responses, etc.). Bytes(Bytes), /// Empty body (HEAD responses, etc.). Empty, + /// Runtime-native streaming body, bypassing Rust stream intermediaries. + /// On CF Workers this is `Option`; + /// on the server runtime it's a reqwest byte stream. + Native(N), } -impl ProxyResponseBody { +impl ProxyResponseBody { /// Create a response body from raw bytes. pub fn from_bytes(bytes: Bytes) -> Self { if bytes.is_empty() { diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs index 370fa1c..1700cf7 100644 --- a/crates/runtimes/cf-workers/src/body.rs +++ b/crates/runtimes/cf-workers/src/body.rs @@ -11,10 +11,12 @@ use worker::{Headers, Response}; /// Build a `worker::Response` from a `ProxyResult`. /// -/// Stream bodies are bridged to JS `ReadableStream` via a `TransformStream`: -/// a spawn_local task reads Rust stream chunks and writes them to the -/// writable side; the readable side is used for the Response. -pub fn build_worker_response(result: ProxyResult) -> Result { +/// - `Native` bodies (JS `ReadableStream`) are passed through directly — zero conversion. +/// - `Stream` bodies (Rust `BoxStream`) are bridged to JS via a `TransformStream`. +/// - `Bytes`/`Empty` are returned as fixed responses. +pub fn build_worker_response( + result: ProxyResult>, +) -> Result { let resp_headers = Headers::new(); for (key, value) in result.headers.iter() { if let Ok(v) = value.to_str() { @@ -23,7 +25,30 @@ pub fn build_worker_response(result: ProxyResult) -> Result { + // Pass the JS ReadableStream directly — no Rust stream conversion! + let ws_headers = web_sys::Headers::new() + .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; + for (key, value) in result.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } + } + + let init = web_sys::ResponseInit::new(); + init.set_status(result.status); + init.set_headers(&ws_headers.into()); + + let ws_response = web_sys::Response::new_with_opt_readable_stream_and_init( + maybe_stream.as_ref(), + &init, + ) + .map_err(|e| { + worker::Error::RustError(format!("failed to build response: {:?}", e)) + })?; + + Ok(ws_response.into()) + } ProxyResponseBody::Stream(stream) => { // Bridge Rust Stream -> JS ReadableStream via TransformStream let transform = web_sys::TransformStream::new() diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index dc0b834..10c98b2 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -8,7 +8,9 @@ use crate::fetch_connector::FetchConnector; use bytes::Bytes; use http::HeaderMap; use object_store::ObjectStore; -use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse, StoreBuilder}; +use s3_proxy_core::backend::{ + build_object_store, ProxyBackend, RawResponse, RawStreamingResponse, StoreBuilder, +}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; use s3_proxy_source_coop::api::{CacheOptions, HttpClient}; @@ -109,6 +111,35 @@ impl HttpClient for WorkerHttpClient { } } +/// Headers to extract from backend streaming responses. +const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ + "content-type", + "content-length", + "content-range", + "etag", + "last-modified", + "accept-ranges", + "content-encoding", + "content-disposition", + "cache-control", + "x-amz-request-id", + "x-amz-version-id", + "location", +]; + +/// Extract response headers from a `web_sys::Headers` using an allowlist. +fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { + let mut resp_headers = HeaderMap::new(); + for name in RESPONSE_HEADER_ALLOWLIST { + if let Ok(Some(value)) = ws_headers.get(name) { + if let Ok(parsed) = value.parse() { + resp_headers.insert(*name, parsed); + } + } + } + resp_headers +} + /// Backend for the Cloudflare Workers runtime. /// /// Uses `FetchConnector` for `object_store` HTTP requests and `web_sys::fetch` @@ -117,6 +148,8 @@ impl HttpClient for WorkerHttpClient { pub struct WorkerBackend; impl ProxyBackend for WorkerBackend { + type NativeBody = Option; + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { build_object_store(config, |b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), @@ -178,23 +211,7 @@ impl ProxyBackend for WorkerBackend { // Convert response headers let ws_response: web_sys::Response = worker_resp.into(); - let mut resp_headers = HeaderMap::new(); - let response_headers = ws_response.headers(); - for name in &[ - "content-type", - "content-length", - "etag", - "last-modified", - "x-amz-request-id", - "x-amz-version-id", - "location", - ] { - if let Ok(Some(value)) = response_headers.get(name) { - if let Ok(parsed) = value.parse() { - resp_headers.insert(*name, parsed); - } - } - } + let resp_headers = extract_response_headers(&ws_response.headers()); Ok(RawResponse { status, @@ -202,4 +219,58 @@ impl ProxyBackend for WorkerBackend { body: Bytes::from(resp_bytes), }) } + + async fn send_streaming( + &self, + method: http::Method, + url: String, + headers: HeaderMap, + ) -> Result, ProxyError> { + tracing::debug!( + method = %method, + url = %url, + "worker: sending streaming backend request via Fetch API" + ); + + // Build web_sys::Headers + let ws_headers = web_sys::Headers::new() + .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; + + for (key, value) in headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } + } + + // Build web_sys::RequestInit + let init = web_sys::RequestInit::new(); + init.set_method(method.as_str()); + init.set_headers(&ws_headers.into()); + + let ws_request = + web_sys::Request::new_with_str_and_init(&url, &init).map_err(|e| { + ProxyError::BackendError(format!("failed to create request: {:?}", e)) + })?; + + // Fetch via worker — returns a JS Response with a ReadableStream body + let worker_req: worker::Request = ws_request.into(); + let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { + ProxyError::BackendError(format!("fetch failed: {}", e)) + })?; + + let status = worker_resp.status_code(); + + // Extract response headers and body stream from the web_sys::Response + let ws_response: web_sys::Response = worker_resp.into(); + let resp_headers = extract_response_headers(&ws_response.headers()); + + // Get the ReadableStream directly — no conversion to Rust types! + let body = ws_response.body(); + + Ok(RawStreamingResponse { + status, + headers: resp_headers, + body, + }) + } } diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index a0814b4..bee9bec 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -200,6 +200,10 @@ fn convert_headers(req: &Request) -> http::HeaderMap { "content-length", "content-md5", "range", + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", ] { if let Ok(Some(value)) = req.headers().get(name) { if let Ok(parsed) = value.parse() { diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs index 37b3bb8..bc552ca 100644 --- a/crates/runtimes/server/src/body.rs +++ b/crates/runtimes/server/src/body.rs @@ -3,12 +3,16 @@ //! Converts [`ProxyResponseBody`] to a streaming hyper response body. use bytes::Bytes; -use futures::TryStreamExt; +use futures::{Stream, TryStreamExt}; use http::Response; use http_body_util::{Either, Empty, Full, StreamBody}; use hyper::body::Frame; use s3_proxy_core::proxy::ProxyResult; use s3_proxy_core::response_body::ProxyResponseBody; +use std::pin::Pin; + +/// The server's native body type from `send_streaming`. +type NativeStream = Pin> + Send>>; /// A boxed streaming body type that erases concrete stream types. type BoxedStreamBody = StreamBody< @@ -21,11 +25,8 @@ type BoxedStreamBody = StreamBody< pub type ServerResponseBody = Either, Empty>>; /// Convert a `ProxyResult` to a hyper `Response` with a streaming body. -/// -/// This is an improvement over the old implementation: GET responses now -/// stream through without buffering the entire body in memory. pub fn build_hyper_response( - result: ProxyResult, + result: ProxyResult, ) -> Result, Box> { let mut builder = Response::builder().status(result.status); @@ -34,6 +35,12 @@ pub fn build_hyper_response( } let body = match result.body { + ProxyResponseBody::Native(stream) => { + let framed = stream + .map_ok(Frame::data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); + Either::Left(StreamBody::new(Box::pin(framed) as std::pin::Pin, std::io::Error>> + Send>>)) + } ProxyResponseBody::Stream(stream) => { let framed = stream .map_ok(Frame::data) diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index 8f8d36b..fd175bc 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -1,11 +1,13 @@ //! Server backend using reqwest for raw HTTP and default object_store connector. use bytes::Bytes; +use futures::Stream; use http::HeaderMap; use object_store::ObjectStore; -use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse}; +use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse, RawStreamingResponse}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; +use std::pin::Pin; use std::sync::Arc; /// Backend for the Tokio/Hyper server runtime. @@ -35,6 +37,8 @@ impl Default for ServerBackend { } impl ProxyBackend for ServerBackend { + type NativeBody = Pin> + Send>>; + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { build_object_store(config, |b| b) } @@ -79,4 +83,38 @@ impl ProxyBackend for ServerBackend { body: resp_body, }) } + + async fn send_streaming( + &self, + method: http::Method, + url: String, + headers: HeaderMap, + ) -> Result, ProxyError> { + tracing::debug!( + method = %method, + url = %url, + "server: sending streaming backend request via reqwest" + ); + + let mut req_builder = self.client.request(method, &url); + + for (key, value) in headers.iter() { + req_builder = req_builder.header(key, value); + } + + let response = req_builder.send().await.map_err(|e| { + tracing::error!(error = %e, "reqwest streaming request failed"); + ProxyError::BackendError(e.to_string()) + })?; + + let status = response.status().as_u16(); + let resp_headers = response.headers().clone(); + let body = Box::pin(response.bytes_stream()); + + Ok(RawStreamingResponse { + status, + headers: resp_headers, + body, + }) + } } From 51c4d6a8df64fbb9389281a080ae9ba4e441bf4c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 14:13:01 -0500 Subject: [PATCH 16/82] refactor: use signed URLs to simplify how we stream data from various runtimes --- CLAUDE.md | 34 +- README.md | 11 +- crates/libs/core/README.md | 12 +- crates/libs/core/src/backend.rs | 207 ++++++-- crates/libs/core/src/proxy.rs | 589 ++++++++--------------- crates/libs/core/src/response_body.rs | 24 +- crates/runtimes/cf-workers/README.md | 25 +- crates/runtimes/cf-workers/src/body.rs | 90 +--- crates/runtimes/cf-workers/src/client.rs | 113 ++--- crates/runtimes/cf-workers/src/lib.rs | 132 ++++- crates/runtimes/server/README.md | 20 +- crates/runtimes/server/src/body.rs | 33 +- crates/runtimes/server/src/client.rs | 50 +- crates/runtimes/server/src/server.rs | 103 +++- 14 files changed, 675 insertions(+), 768 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 718402c..ae96a87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # S3 Proxy Gateway -Multi-runtime S3 gateway proxy in Rust. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. Uses the `object_store` crate for high-level operations (GET, HEAD, PUT, LIST) and raw signed HTTP for multipart uploads. +Multi-runtime S3 gateway proxy in Rust. Each runtime have feature parity. Proxies S3-compatible API requests to backend object stores with authentication, authorization, and streaming passthrough. Uses presigned URLs via `object_store`'s `Signer` trait for GET/HEAD/PUT/DELETE (enabling zero-copy streaming), `object_store` directly for LIST, and raw signed HTTP for multipart uploads. The intention of this codebase is to serve as a data proxy for the Source Cooperative. However, it should be structured in a way for others to use and build upon for their individual proxy needs. As such, a modular approach should be utilized to enable others to compose similar but different sytems. @@ -27,21 +27,23 @@ cargo test ## Key Architecture Notes -- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has two methods: `create_store()` returns an `Arc` for high-level operations, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: - - **Server**: `ServerBackend` delegates to `build_object_store()` with identity connector and uses reqwest for raw HTTP. - - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` injecting `FetchConnector`, and uses `web_sys::fetch` for raw HTTP. -- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. The `build_object_store()` function in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterates `backend_options` calling `with_config()` on the appropriate builder (`AmazonS3Builder`, `MicrosoftAzureBuilder`, `GoogleCloudStorageBuilder`). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder`. -- **Operation dispatch**: The proxy handler dispatches S3 operations to different backends: - - **GET** → `store.get_opts()` with Range/If-Match/If-None-Match header parsing; returns `ProxyResponseBody::Stream`. - - **HEAD** → `store.head()`; returns metadata headers + empty body. - - **PUT** → `store.put()`; request body materialized to `Bytes` by the runtime before calling the handler. - - **LIST** → `store.list_with_delimiter()`; builds S3 ListObjectsV2 XML directly from `ListResult` (no XML rewriting needed). `IsTruncated` is always `false` (object_store fetches all pages internally). - - **Multipart** (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload) → raw signed HTTP via `backend.send_raw()` + `S3RequestSigner`. These use raw HTTP because `object_store`'s `MultipartUpload` API manages state internally and doesn't expose upload IDs for stateless proxying. -- **ProxyResponseBody**: A concrete enum (`Stream`, `Bytes`, `Empty`) replacing the old generic `B: BodyStream` type parameter. Runtimes convert this to their native response type at the edge. `Stream` wraps a `BoxStream<'static, Result>`. -- **RequestResolver pattern**: The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.handle_request()`. -- **MaybeSend pattern**: Core traits use `MaybeSend`/`MaybeSync` (defined in `crates/libs/core/src/maybe_send.rs`) instead of `Send`/`Sync`. On native targets these resolve to `Send`/`Sync`; on `wasm32` they are no-op blanket traits. This allows the CF Workers runtime to use `!Send` JS interop types (`JsValue`, `ReadableStream`, etc.). -- **FetchConnector** (CF Workers): `crates/runtimes/cf-workers/src/fetch_connector.rs` implements `object_store::client::HttpConnector` and `HttpService` using the Workers Fetch API. Since `worker::Fetch::send()` is `!Send`, each call is wrapped in `spawn_local` with a oneshot channel to bridge back to the `Send` context that `object_store` expects. Response body streaming uses an mpsc channel: a `spawn_local` task reads from the Workers `ByteStream` and sends chunks through the channel, whose receiver is wrapped as an `HttpResponseBody`. -- **Streaming**: The server runtime now streams GET responses (previously buffered). The CF Workers runtime bridges `object_store`'s `BoxStream` to a JS `ReadableStream` via `TransformStream` — a `spawn_local` task reads Rust stream chunks and writes them to the writable side; the readable side is returned in the Response. Bytes cross the WASM boundary in chunks (lazy, not buffered). +- **Two-phase handler**: `ProxyHandler::resolve_request()` returns a `HandlerAction` enum: + - `Forward(ForwardRequest)` — presigned URL + headers for GET/HEAD/PUT/DELETE. The runtime executes the request with its native HTTP client, enabling zero-copy streaming. + - `Response(ProxyResult)` — complete response for LIST, errors, synthetic responses. + - `NeedsBody(PendingRequest)` — multipart operations that need the request body. The runtime materializes the body and calls `handle_with_body()`. +- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has three methods: `create_store()` returns an `Arc` for LIST, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: + - **Server**: `ServerBackend` delegates to `build_object_store()` (with default connector) and `build_signer()`, and uses reqwest for raw HTTP + Forward execution. + - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` (injecting `FetchConnector`) and `build_signer()`, and uses `web_sys::fetch` for raw HTTP + Forward execution. +- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_object_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_object_store()` only — `build_signer()` needs no connector since signing is pure computation. +- **Operation dispatch** via presigned URLs and direct object_store: + - **GET/HEAD/PUT/DELETE** → `create_signer()` generates a presigned URL, returned as `HandlerAction::Forward`. The runtime executes the URL with its native HTTP client, streaming request/response bodies directly without handler involvement. + - **LIST** → `create_store()` + `store.list_with_delimiter()`; builds S3 ListObjectsV2 XML from `ListResult`. `IsTruncated` is always `false`. + - **Multipart** (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload) → `NeedsBody` then raw signed HTTP via `backend.send_raw()` + `S3RequestSigner`. +- **ProxyResponseBody**: A simple enum (`Bytes`, `Empty`) for non-streaming responses only. Streaming bodies bypass this type entirely via the `Forward` action — runtimes handle them natively. +- **RequestResolver pattern**: The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.resolve_request()`. +- **MaybeSend pattern**: Core traits use `MaybeSend`/`MaybeSync` (defined in `crates/libs/core/src/maybe_send.rs`) instead of `Send`/`Sync`. On native targets these resolve to `Send`/`Sync`; on `wasm32` they are no-op blanket traits. This allows the CF Workers runtime to use `!Send` JS interop types (`JsValue`, `ReadableStream`, etc.). The `Signer` trait from `object_store` requires real `Send + Sync`, which works because `UnsignedUrlSigner` only holds `String` fields and `object_store`'s built-in store types are `Send + Sync`. +- **FetchConnector** (CF Workers): `crates/runtimes/cf-workers/src/fetch_connector.rs` implements `object_store::client::HttpConnector` and `HttpService` using the Workers Fetch API. Since `worker::Fetch::send()` is `!Send`, each call is wrapped in `spawn_local` with a oneshot channel to bridge back to the `Send` context that `object_store` expects. Only exercised for LIST operations (presigned URL operations bypass `object_store` entirely). +- **Streaming via Forward pattern**: For GET, the runtime sends a presigned URL request and streams the response body directly to the client. For PUT, the runtime streams the client's request body directly to the presigned URL. On CF Workers, JS `ReadableStream` objects pass through without touching Rust. On the server, reqwest streams hyper `Incoming` bodies and `bytes_stream()` responses. - **cf-workers is excluded from `default-members`** in the root `Cargo.toml` because WASM types are `!Send` and will fail to compile on native targets. Always use `--target wasm32-unknown-unknown` when working with this crate. - **Config loading** (CF Workers): `PROXY_CONFIG` can be either a JSON string (via `wrangler secret`) or a JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`). Both formats are handled. - **List response construction**: LIST responses are built directly from `object_store::ListResult` as S3 XML. When a resolver returns a `ListRewrite`, prefix stripping/adding is applied to `ObjectMeta.location` and `common_prefixes` paths before XML generation. The `list_rewrite` module in `crates/libs/core/src/s3/list_rewrite.rs` is retained for backward compatibility. diff --git a/README.md b/README.md index 03de276..d658b03 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ crates/ └── cf-workers/ (s3-proxy-cf-workers) # Cloudflare Workers for edge deployments ``` -Libraries define trait abstractions (`ProxyBackend`, `ConfigProvider`, `RequestResolver`). Runtimes implement `ProxyBackend` with platform-native primitives: the server runtime uses `object_store` with its default HTTP connector and `reqwest` for raw multipart requests; the Workers runtime uses a custom `FetchConnector` that bridges `object_store` to the JS Fetch API. +Libraries define trait abstractions (`ProxyBackend`, `ConfigProvider`, `RequestResolver`). Runtimes implement `ProxyBackend` with platform-native primitives. The handler uses a two-phase dispatch model: `resolve_request()` returns a `HandlerAction` — either a `Forward` (presigned URL for GET/HEAD/PUT/DELETE), a `Response` (LIST, errors), or `NeedsBody` (multipart). Runtimes execute `Forward` requests with their native HTTP client, enabling zero-copy streaming. The `RequestResolver` trait decouples "what to do with a request" from the proxy handler. A `DefaultResolver` handles standard S3 proxy behavior (parse, auth, authorize via `ConfigProvider`), while custom resolvers like `SourceCoopResolver` can implement entirely different namespace mapping and authorization schemes. @@ -248,7 +248,8 @@ Wire it into the proxy handler in your runtime: ```rust let resolver = MyResolver::new(/* ... */); let handler = ProxyHandler::new(backend, resolver); -let result = handler.handle_request(method, path, query, &headers, body).await; +let action = handler.resolve_request(method, path, query, &headers).await; +// Handle action: Forward (presigned URL), Response, or NeedsBody (multipart) ``` ### Caching Configuration @@ -316,11 +317,11 @@ The proxy validates the JWT against the OIDC provider's JWKS, checks the trust p The crate workspace separates concerns so the core logic compiles to both native and WASM targets: -**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. It uses `object_store` for high-level operations (GET, HEAD, PUT, LIST) and a `ProxyBackend` trait for runtime-specific store creation and raw HTTP (multipart). All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. +**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. It uses `object_store`'s `Signer` trait to generate presigned URLs for GET/HEAD/PUT/DELETE, and `object_store` directly for LIST. A `ProxyBackend` trait provides runtime-specific store/signer creation and raw HTTP (multipart). All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. -**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It implements `ProxyBackend` using `object_store`'s default HTTP connector for high-level operations and reqwest for raw multipart requests. GET responses stream from `object_store` through Hyper without buffering. +**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It handles `Forward` actions by executing presigned URLs via reqwest — streaming the Hyper `Incoming` body for PUT and the reqwest `bytes_stream()` for GET responses. No buffering. Multipart and LIST go through the handler's `Response` path. -**`s3-proxy-cf-workers`** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It implements `ProxyBackend` with a custom `FetchConnector` that bridges `object_store` to the Workers Fetch API via `spawn_local` + channel patterns (since JS interop types are `!Send`). Response body streams are converted to JS `ReadableStream` via `TransformStream` for efficient delivery. +**`s3-proxy-cf-workers`** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It handles `Forward` actions by passing JS `ReadableStream` bodies directly through the Fetch API — zero Rust stream involvement. `FetchConnector` bridges `object_store` to the Workers Fetch API (used only for LIST). ## License diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md index b24cd2d..359e1f6 100644 --- a/crates/libs/core/README.md +++ b/crates/libs/core/README.md @@ -10,7 +10,7 @@ The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in conta The core defines three trait boundaries that runtime crates implement: -**`ProxyBackend`** — Creates `object_store` instances for high-level S3 operations (GET, HEAD, PUT, LIST) and provides a `send_raw()` method for multipart operations that require signed HTTP requests. The server runtime uses `object_store`'s default HTTP connector + reqwest; the worker runtime uses a custom `FetchConnector` that bridges to the JS Fetch API. +**`ProxyBackend`** — Provides three capabilities: `create_store()` returns an `ObjectStore` for LIST, `create_signer()` returns a `Signer` for presigned URL generation (GET/HEAD/PUT/DELETE), and `send_raw()` sends signed HTTP requests for multipart operations. Both runtimes delegate to `build_signer()` which uses `object_store`'s built-in signer for authenticated backends and `UnsignedUrlSigner` for anonymous backends (avoiding `Instant::now()` which panics on WASM). For `create_store()`, the server runtime uses default connectors + reqwest; the worker runtime uses a custom `FetchConnector`. **`ConfigProvider`** — Retrieves bucket, role, and credential configuration. Ships with four implementations behind feature flags: @@ -32,7 +32,7 @@ Any provider can be wrapped with `CachedProvider` for in-memory TTL caching. ``` src/ ├── auth.rs SigV4 verification, identity resolution, authorization -├── backend.rs ProxyBackend trait, S3RequestSigner (multipart), RawResponse +├── backend.rs ProxyBackend trait, Signer/StoreBuilder, S3RequestSigner (multipart) ├── config/ │ ├── mod.rs ConfigProvider trait definition │ ├── cached.rs TTL caching wrapper for any provider @@ -47,7 +47,7 @@ src/ │ ├── request.rs Parse incoming HTTP → S3Operation enum │ ├── response.rs Serialize S3 XML responses │ └── list_rewrite.rs Rewrite / values in list response XML -├── response_body.rs ProxyResponseBody enum (Stream, Bytes, Empty) +├── response_body.rs ProxyResponseBody enum (Bytes, Empty) └── types.rs BucketConfig, RoleConfig, StoredCredential, etc. ``` @@ -71,8 +71,8 @@ let resolver = DefaultResolver::new(config, Some("s3.example.com".into())); let handler = ProxyHandler::new(backend, resolver); // In your HTTP handler: -let result = handler.handle_request(method, path, query, &headers, body).await; -// Convert `result` (ProxyResult with ProxyResponseBody) to your runtime's HTTP response. +let action = handler.resolve_request(method, path, query, &headers).await; +// Handle action: Forward (presigned URL), Response (ProxyResult), or NeedsBody (multipart) ``` ### Custom resolver @@ -104,7 +104,7 @@ impl RequestResolver for MyResolver { let handler = ProxyHandler::new(backend, MyResolver::new()); ``` -See `s3-proxy-cf-workers/src/source_resolver.rs` for a real-world example that maps a `/{account}/{repo}/{key}` namespace to dynamically-resolved S3 backends with external API authorization. +See `crates/libs/source-coop/src/resolver.rs` for a real-world example that maps a `/{account}/{repo}/{key}` namespace to dynamically-resolved S3 backends with external API authorization. ## Feature Flags diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index 73c9403..d6272b9 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -1,16 +1,21 @@ //! Backend abstraction for proxying requests to backing object stores. //! -//! [`ProxyBackend`] is the main trait runtimes implement. It provides two +//! [`ProxyBackend`] is the main trait runtimes implement. It provides three //! capabilities: //! //! 1. **`create_store()`** — build an `ObjectStore` for high-level operations -//! (GET, HEAD, PUT, LIST) routed through `object_store`. -//! 2. **`send_raw()`** — send a pre-signed HTTP request for operations not +//! (LIST) routed through `object_store`. +//! 2. **`create_signer()`** — build a `Signer` for generating presigned URLs +//! for GET, HEAD, PUT, DELETE operations. +//! 3. **`send_raw()`** — send a pre-signed HTTP request for operations not //! covered by `ObjectStore` (multipart uploads). //! //! [`S3RequestSigner`] is retained for signing multipart requests. -//! [`build_object_store`] dispatches on `BucketConfig::backend_type` to build -//! the appropriate provider store. +//! [`build_object_store`] and [`build_signer`] dispatch on +//! `BucketConfig::backend_type` to build the appropriate provider. +//! [`build_signer`] uses `object_store`'s built-in signer for authenticated +//! backends, and [`UnsignedUrlSigner`] for anonymous backends (avoiding +//! `Instant::now()` which panics on WASM). use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; @@ -18,6 +23,7 @@ use crate::types::{BackendType, BucketConfig}; use bytes::Bytes; use http::HeaderMap; use object_store::aws::AmazonS3Builder; +use object_store::signer::Signer; use object_store::ObjectStore; use std::future::Future; use std::sync::Arc; @@ -33,15 +39,19 @@ use object_store::gcp::GoogleCloudStorageBuilder; /// - Server runtime: uses `reqwest` for raw HTTP, default `object_store` HTTP connector /// - Worker runtime: uses `web_sys::fetch` for raw HTTP, custom `FetchConnector` for `object_store` pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { - /// The runtime's native streaming body type. - /// - /// CF Workers uses `Option` to avoid JS/Rust - /// stream round-trips. The server runtime uses a reqwest byte stream. - type NativeBody: MaybeSend + 'static; - /// Create an `ObjectStore` for the given bucket configuration. + /// + /// Used for LIST operations where `object_store` handles pagination + /// and response parsing internally. fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; + /// Create a `Signer` for generating presigned URLs. + /// + /// Used for GET, HEAD, PUT, DELETE operations. The handler generates + /// a presigned URL and the runtime executes the request with its + /// native HTTP client, enabling zero-copy streaming. + fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError>; + /// Send a raw HTTP request (used for multipart operations that /// `ObjectStore` doesn't expose at the right abstraction level). fn send_raw( @@ -51,17 +61,6 @@ pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { headers: HeaderMap, body: Bytes, ) -> impl Future> + MaybeSend; - - /// Send a raw HTTP request and return the response with a streaming body. - /// - /// Used for GET/HEAD on S3 backends to avoid double stream conversion - /// (JS→Rust→JS on Workers, or unnecessary buffering on server). - fn send_streaming( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - ) -> impl Future, ProxyError>> + MaybeSend; } /// Response from a raw HTTP request to a backend. @@ -71,20 +70,10 @@ pub struct RawResponse { pub body: Bytes, } -/// Response from a streaming HTTP request to a backend. -/// -/// The body is the runtime's native streaming type, avoiding -/// intermediate conversion through Rust stream types. -pub struct RawStreamingResponse { - pub status: u16, - pub headers: HeaderMap, - pub body: N, -} - /// Wrapper around provider-specific `object_store` builders. /// -/// Runtimes use [`build_object_store`] and inject their HTTP connector via -/// a closure that receives this enum. +/// Runtimes use [`build_object_store`] and inject their HTTP connector +/// via a closure that receives this enum. pub enum StoreBuilder { S3(AmazonS3Builder), #[cfg(feature = "azure")] @@ -113,22 +102,35 @@ impl StoreBuilder { )), } } + + /// Build a `Signer` for presigned URL generation. + pub fn build_signer(self) -> Result, ProxyError> { + match self { + StoreBuilder::S3(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 signer: {}", e)))?, + )), + #[cfg(feature = "azure")] + StoreBuilder::Azure(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build Azure signer: {}", e)))?, + )), + #[cfg(feature = "gcp")] + StoreBuilder::Gcs(b) => Ok(Arc::new( + b.build() + .map_err(|e| ProxyError::ConfigError(format!("failed to build GCS signer: {}", e)))?, + )), + } + } } -/// Build an `ObjectStore` from a [`BucketConfig`], dispatching on `backend_type`. -/// -/// The `configure` closure lets each runtime inject its HTTP connector: -/// - Server runtime passes `|b| b` (default connector) -/// - CF Workers passes `|b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), .. }` -pub fn build_object_store(config: &BucketConfig, configure: F) -> Result, ProxyError> -where - F: FnOnce(StoreBuilder) -> StoreBuilder, -{ +/// Create a [`StoreBuilder`] from a [`BucketConfig`], dispatching on `backend_type`. +fn create_builder(config: &BucketConfig) -> Result { let backend_type = config .parsed_backend_type() .ok_or_else(|| ProxyError::ConfigError(format!("unsupported backend_type: '{}'", config.backend_type)))?; - let builder = match backend_type { + match backend_type { BackendType::S3 => { let mut b = AmazonS3Builder::new(); for (k, v) in &config.backend_options { @@ -136,7 +138,7 @@ where b = b.with_config(key, v); } } - StoreBuilder::S3(b) + Ok(StoreBuilder::S3(b)) } #[cfg(feature = "azure")] BackendType::Azure => { @@ -146,13 +148,13 @@ where b = b.with_config(key, v); } } - StoreBuilder::Azure(b) + Ok(StoreBuilder::Azure(b)) } #[cfg(not(feature = "azure"))] BackendType::Azure => { - return Err(ProxyError::ConfigError( + Err(ProxyError::ConfigError( "Azure backend support not enabled (requires 'azure' feature)".into(), - )); + )) } #[cfg(feature = "gcp")] BackendType::Gcs => { @@ -162,17 +164,67 @@ where b = b.with_config(key, v); } } - StoreBuilder::Gcs(b) + Ok(StoreBuilder::Gcs(b)) } #[cfg(not(feature = "gcp"))] BackendType::Gcs => { - return Err(ProxyError::ConfigError( + Err(ProxyError::ConfigError( "GCS backend support not enabled (requires 'gcp' feature)".into(), - )); + )) } - }; + } +} - configure(builder).build() +/// Build an `ObjectStore` from a [`BucketConfig`], dispatching on `backend_type`. +/// +/// The `configure` closure lets each runtime inject its HTTP connector: +/// - Server runtime passes `|b| b` (default connector) +/// - CF Workers passes `|b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), .. }` +pub fn build_object_store(config: &BucketConfig, configure: F) -> Result, ProxyError> +where + F: FnOnce(StoreBuilder) -> StoreBuilder, +{ + configure(create_builder(config)?).build() +} + +/// Build a [`Signer`] from a [`BucketConfig`], dispatching on `backend_type`. +/// +/// For backends with credentials, uses `object_store`'s built-in signer +/// (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`). +/// For anonymous backends (no credentials), returns [`UnsignedUrlSigner`] +/// which constructs plain URLs without auth parameters, avoiding the +/// `InstanceCredentialProvider` → `Instant::now()` panic on WASM. +pub fn build_signer(config: &BucketConfig) -> Result, ProxyError> { + let backend_type = config + .parsed_backend_type() + .ok_or_else(|| { + ProxyError::ConfigError(format!("unsupported backend_type: '{}'", config.backend_type)) + })?; + + // Check for credentials — if absent, return unsigned signer to avoid + // InstanceCredentialProvider which uses Instant::now() (panics on WASM). + let has_creds = !config.option("access_key_id").unwrap_or("").is_empty() + && !config.option("secret_access_key").unwrap_or("").is_empty(); + + if !has_creds { + return Ok(Arc::new(UnsignedUrlSigner::from_config(config)?)); + } + + match backend_type { + BackendType::S3 => create_builder(config)?.build_signer(), + #[cfg(feature = "azure")] + BackendType::Azure => create_builder(config)?.build_signer(), + #[cfg(not(feature = "azure"))] + BackendType::Azure => Err(ProxyError::ConfigError( + "Azure backend support not enabled (requires 'azure' feature)".into(), + )), + #[cfg(feature = "gcp")] + BackendType::Gcs => create_builder(config)?.build_signer(), + #[cfg(not(feature = "gcp"))] + BackendType::Gcs => Err(ProxyError::ConfigError( + "GCS backend support not enabled (requires 'gcp' feature)".into(), + )), + } } /// Helper to build a signed URL + headers for an outbound request to S3. @@ -316,6 +368,55 @@ impl S3RequestSigner { } } +/// Signer for anonymous/credential-less backends. +/// +/// Returns unsigned URLs — no auth query params, no time calls. This avoids +/// the `InstanceCredentialProvider` → `TokenCache` → `Instant::now()` path +/// in `object_store` which panics on `wasm32-unknown-unknown`. +#[derive(Debug)] +struct UnsignedUrlSigner { + endpoint: String, + bucket: String, +} + +impl UnsignedUrlSigner { + fn from_config(config: &BucketConfig) -> Result { + let endpoint = config.option("endpoint").unwrap_or("https://s3.amazonaws.com"); + let bucket = config.option("bucket_name").unwrap_or(""); + Ok(Self { + endpoint: endpoint.trim_end_matches('/').to_string(), + bucket: bucket.to_string(), + }) + } +} + +#[async_trait::async_trait] +impl Signer for UnsignedUrlSigner { + async fn signed_url( + &self, + _method: http::Method, + path: &object_store::path::Path, + _expires_in: std::time::Duration, + ) -> object_store::Result { + let key = path.as_ref(); + let url_str = if self.bucket.is_empty() { + if key.is_empty() { + format!("{}/", self.endpoint) + } else { + format!("{}/{}", self.endpoint, key) + } + } else if key.is_empty() { + format!("{}/{}", self.endpoint, self.bucket) + } else { + format!("{}/{}/{}", self.endpoint, self.bucket, key) + }; + url::Url::parse(&url_str).map_err(|e| object_store::Error::Generic { + store: "UnsignedUrlSigner", + source: Box::new(e), + }) + } +} + /// Hash a payload for SigV4. For streaming/unsigned payloads, use the /// special sentinel value. pub fn hash_payload(payload: &[u8]) -> String { diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index bc55d50..ea171c3 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -1,13 +1,22 @@ //! The main proxy handler that ties together resolution and backend forwarding. //! //! [`ProxyHandler`] is generic over the runtime's backend and request resolver. -//! GET/HEAD operations use `send_streaming` for S3 backends (avoiding double -//! stream conversion) and `object_store` for non-S3 backends. Multipart -//! operations use raw signed HTTP requests. +//! It uses a two-phase dispatch model: +//! +//! 1. **`resolve_request`** — parses, authenticates, and decides the action: +//! - GET/HEAD/PUT/DELETE → [`HandlerAction::Forward`] with a presigned URL +//! - LIST → [`HandlerAction::Response`] with XML body +//! - Multipart → [`HandlerAction::NeedsBody`] (body required) +//! - Errors/synthetic → [`HandlerAction::Response`] +//! +//! 2. **`handle_with_body`** — completes multipart operations once the body arrives. +//! +//! Runtimes handle [`Forward`] by executing the presigned URL with their native +//! HTTP client, enabling zero-copy streaming for both request and response bodies. use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLOAD}; use crate::error::ProxyError; -use crate::resolver::{ListRewrite, ResolvedAction, RequestResolver}; +use crate::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use crate::response_body::ProxyResponseBody; use crate::s3::response::{ ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents, @@ -15,15 +24,50 @@ use crate::s3::response::{ use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; -use object_store::{GetOptions, GetRange, ObjectStore, PutPayload}; +use object_store::ObjectStore; +use std::time::Duration; use url::Url; use uuid::Uuid; +/// TTL for presigned URLs. Short because they're used immediately. +const PRESIGNED_URL_TTL: Duration = Duration::from_secs(300); + +/// The action the handler wants the runtime to take. +pub enum HandlerAction { + /// A fully formed response (LIST results, errors, synthetic responses). + Response(ProxyResult), + /// A presigned URL for the runtime to execute with its native HTTP client. + /// The runtime streams request/response bodies directly — no handler involvement. + Forward(ForwardRequest), + /// The handler needs the request body to continue (multipart operations). + /// The runtime should materialize the body and call `handle_with_body`. + NeedsBody(PendingRequest), +} + +/// A presigned URL request for the runtime to execute. +pub struct ForwardRequest { + /// HTTP method for the backend request. + pub method: Method, + /// Presigned URL to the backend (includes auth in query params). + pub url: Url, + /// Headers to include in the backend request (Range, If-Match, Content-Type, etc.). + pub headers: HeaderMap, +} + +/// Opaque state for a multipart operation that needs the request body. +pub struct PendingRequest { + method: Method, + operation: S3Operation, + bucket_config: BucketConfig, + original_headers: HeaderMap, + request_id: String, +} + /// The core proxy handler, generic over runtime primitives. /// /// # Type Parameters /// -/// - `B`: The runtime's backend for object store creation and raw HTTP +/// - `B`: The runtime's backend for object store creation, signing, and raw HTTP /// - `R`: The request resolver that decides what action to take for each request pub struct ProxyHandler { backend: B, @@ -39,19 +83,21 @@ where Self { backend, resolver } } - /// Handle an incoming S3 request. + /// Phase 1: Resolve an incoming request into an action. /// /// This is the main entry point. It: /// 1. Resolves the request via the resolver (parse, auth, authorize) - /// 2. Forwards the request to the backing store or returns a synthetic response - pub async fn handle_request( + /// 2. Determines what the runtime should do next: + /// - `Forward` a presigned URL (GET/HEAD/PUT/DELETE) + /// - Return a `Response` directly (LIST, errors, synthetic) + /// - Request the body via `NeedsBody` (multipart) + pub async fn resolve_request( &self, method: Method, path: &str, query: Option<&str>, headers: &HeaderMap, - body: Bytes, - ) -> ProxyResult { + ) -> HandlerAction { let request_id = Uuid::new_v4().to_string(); tracing::info!( @@ -63,16 +109,33 @@ where ); match self - .handle_inner(method, path, query, headers, body) + .resolve_inner(method, path, query, headers, &request_id) .await { - Ok(resp) => { - tracing::info!( - request_id = %request_id, - status = resp.status, - "request completed" - ); - resp + Ok(action) => { + match &action { + HandlerAction::Response(resp) => { + tracing::info!( + request_id = %request_id, + status = resp.status, + "request completed" + ); + } + HandlerAction::Forward(fwd) => { + tracing::info!( + request_id = %request_id, + method = %fwd.method, + "forwarding via presigned URL" + ); + } + HandlerAction::NeedsBody(_) => { + tracing::debug!( + request_id = %request_id, + "request needs body (multipart)" + ); + } + } + action } Err(err) => { tracing::warn!( @@ -82,19 +145,49 @@ where s3_code = %err.s3_error_code(), "request failed" ); - error_response(&err, path, &request_id) + HandlerAction::Response(error_response(&err, path, &request_id)) } } } - async fn handle_inner( + /// Phase 2: Complete a multipart operation with the request body. + /// + /// Called by the runtime after materializing the body for a `NeedsBody` action. + pub async fn handle_with_body( + &self, + pending: PendingRequest, + body: Bytes, + ) -> ProxyResult { + match self.execute_multipart(&pending, body).await { + Ok(result) => { + tracing::info!( + request_id = %pending.request_id, + status = result.status, + "multipart request completed" + ); + result + } + Err(err) => { + tracing::warn!( + request_id = %pending.request_id, + error = %err, + status = err.status_code(), + s3_code = %err.s3_error_code(), + "multipart request failed" + ); + error_response(&err, pending.operation.key(), &pending.request_id) + } + } + } + + async fn resolve_inner( &self, method: Method, path: &str, query: Option<&str>, headers: &HeaderMap, - body: Bytes, - ) -> Result, ProxyError> { + request_id: &str, + ) -> Result { let action = self.resolver.resolve(&method, path, query, headers).await?; match action { @@ -102,56 +195,90 @@ where status, headers: resp_headers, body: resp_body, - } => Ok(ProxyResult { + } => Ok(HandlerAction::Response(ProxyResult { status, headers: resp_headers, body: ProxyResponseBody::from_bytes(resp_body), - }), + })), ResolvedAction::Proxy { operation, bucket_config, list_rewrite, } => { - self.forward_to_backend( + self.dispatch_operation( &method, &operation, &bucket_config, headers, - body, list_rewrite.as_ref(), + request_id, ) .await } } } - async fn forward_to_backend( + async fn dispatch_operation( &self, method: &Method, operation: &S3Operation, bucket_config: &BucketConfig, original_headers: &HeaderMap, - body: Bytes, list_rewrite: Option<&ListRewrite>, - ) -> Result, ProxyError> { + request_id: &str, + ) -> Result { match operation { S3Operation::GetObject { key, .. } => { - self.handle_get(bucket_config, key, original_headers).await + let fwd = self.build_forward( + Method::GET, + bucket_config, + key, + original_headers, + &["range", "if-match", "if-none-match", "if-modified-since", "if-unmodified-since"], + ).await?; + tracing::debug!(url = %fwd.url, "GET via presigned URL"); + Ok(HandlerAction::Forward(fwd)) } S3Operation::HeadObject { key, .. } => { - self.handle_head(bucket_config, key, original_headers).await + let fwd = self.build_forward( + Method::HEAD, + bucket_config, + key, + original_headers, + &["if-match", "if-none-match", "if-modified-since", "if-unmodified-since"], + ).await?; + tracing::debug!(url = %fwd.url, "HEAD via presigned URL"); + Ok(HandlerAction::Forward(fwd)) } S3Operation::PutObject { key, .. } => { - self.handle_put(bucket_config, key, body).await + let fwd = self.build_forward( + Method::PUT, + bucket_config, + key, + original_headers, + &["content-type", "content-length", "content-md5"], + ).await?; + tracing::debug!(url = %fwd.url, "PUT via presigned URL"); + Ok(HandlerAction::Forward(fwd)) } S3Operation::DeleteObject { key, .. } => { - self.handle_delete(bucket_config, key).await + let fwd = self.build_forward( + Method::DELETE, + bucket_config, + key, + original_headers, + &[], + ).await?; + tracing::debug!(url = %fwd.url, "DELETE via presigned URL"); + Ok(HandlerAction::Forward(fwd)) } S3Operation::ListBucket { raw_query, .. } => { - self.handle_list(bucket_config, raw_query.as_deref(), list_rewrite) - .await + let result = self + .handle_list(bucket_config, raw_query.as_deref(), list_rewrite) + .await?; + Ok(HandlerAction::Response(result)) } - // Multipart operations go through raw signed HTTP (S3 only) + // Multipart operations need the request body S3Operation::CreateMultipartUpload { .. } | S3Operation::UploadPart { .. } | S3Operation::CompleteMultipartUpload { .. } @@ -162,322 +289,46 @@ where bucket_config.backend_type ))); } - self.handle_multipart(method, operation, bucket_config, original_headers, body) - .await + Ok(HandlerAction::NeedsBody(PendingRequest { + method: method.clone(), + operation: operation.clone(), + bucket_config: bucket_config.clone(), + original_headers: original_headers.clone(), + request_id: request_id.to_string(), + })) } _ => Err(ProxyError::Internal("unexpected operation".into())), } } - /// GET — uses `send_streaming` for S3 backends (zero-copy native body), - /// falls back to `object_store` for non-S3 backends. - async fn handle_get( - &self, - config: &BucketConfig, - key: &str, - headers: &HeaderMap, - ) -> Result, ProxyError> { - if config.supports_s3_multipart() { - return self.handle_get_s3(config, key, headers).await; - } - self.handle_get_object_store(config, key, headers).await - } - - /// GET for S3 backends via `send_streaming` — the response body stays in - /// the runtime's native type, avoiding double JS/Rust stream conversion. - async fn handle_get_s3( - &self, - config: &BucketConfig, - key: &str, - headers: &HeaderMap, - ) -> Result, ProxyError> { - let operation = S3Operation::GetObject { - bucket: String::new(), - key: key.to_string(), - }; - let backend_url = build_backend_url(config, &operation)?; - - let mut req_headers = HeaderMap::new(); - - // Forward conditional and range headers - for header_name in &[ - "range", - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", - ] { - if let Some(val) = headers.get(*header_name) { - req_headers.insert(*header_name, val.clone()); - } - } - - sign_s3_request(&Method::GET, &backend_url, &mut req_headers, config, UNSIGNED_PAYLOAD)?; - - tracing::debug!(backend_url = %backend_url, "GET via send_streaming (S3)"); - - let raw = self.backend.send_streaming(Method::GET, backend_url, req_headers).await?; - - // Forward response headers from the backend - let mut resp_headers = HeaderMap::new(); - for header_name in STREAMING_RESPONSE_HEADERS { - if let Some(val) = raw.headers.get(*header_name) { - resp_headers.insert(*header_name, val.clone()); - } - } - - Ok(ProxyResult { - status: raw.status, - headers: resp_headers, - body: ProxyResponseBody::Native(raw.body), - }) - } - - /// GET for non-S3 backends via `object_store`. - async fn handle_get_object_store( + /// Build a [`ForwardRequest`] with a presigned URL for the given operation. + async fn build_forward( &self, + method: Method, config: &BucketConfig, key: &str, - headers: &HeaderMap, - ) -> Result, ProxyError> { - let store = self.backend.create_store(config)?; + original_headers: &HeaderMap, + forward_header_names: &[&'static str], + ) -> Result { + let signer = self.backend.create_signer(config)?; let path = build_object_path(config, key); - let mut opts = GetOptions::default(); - - // Parse conditional headers - if let Some(val) = headers.get("if-match").and_then(|v| v.to_str().ok()) { - opts.if_match = Some(val.to_string()); - } - if let Some(val) = headers.get("if-none-match").and_then(|v| v.to_str().ok()) { - opts.if_none_match = Some(val.to_string()); - } - if let Some(val) = headers.get("if-modified-since").and_then(|v| v.to_str().ok()) { - if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(val) { - opts.if_modified_since = Some(dt.with_timezone(&chrono::Utc)); - } - } - if let Some(val) = headers - .get("if-unmodified-since") - .and_then(|v| v.to_str().ok()) - { - if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(val) { - opts.if_unmodified_since = Some(dt.with_timezone(&chrono::Utc)); - } - } - - // Parse Range header - if let Some(range_val) = headers.get("range").and_then(|v| v.to_str().ok()) { - if let Some(range) = parse_range_header(range_val) { - opts.range = Some(range); - } - } - - tracing::debug!(path = %path, "GET via object_store"); - - let result = store - .get_opts(&path, opts) + let url = signer + .signed_url(method.clone(), &path, PRESIGNED_URL_TTL) .await .map_err(ProxyError::from_object_store_error)?; - // Build response headers from metadata - let mut resp_headers = HeaderMap::new(); - if let Some(etag) = &result.meta.e_tag { - resp_headers.insert("etag", etag.parse().unwrap()); - } - resp_headers.insert( - "last-modified", - result - .meta - .last_modified - .format("%a, %d %b %Y %H:%M:%S GMT") - .to_string() - .parse() - .unwrap(), - ); - let content_length = result.range.end - result.range.start; - resp_headers.insert("content-length", content_length.to_string().parse().unwrap()); - resp_headers.insert("accept-ranges", "bytes".parse().unwrap()); - - // If this is a range response, set 206 + Content-Range - let status = if result.range.start > 0 - || result.range.end < result.meta.size - { - resp_headers.insert( - "content-range", - format!( - "bytes {}-{}/{}", - result.range.start, - result.range.end.saturating_sub(1), - result.meta.size - ) - .parse() - .unwrap(), - ); - 206 - } else { - 200 - }; - - let stream = result.into_stream(); - - Ok(ProxyResult { - status, - headers: resp_headers, - body: ProxyResponseBody::Stream(stream), - }) - } - - /// HEAD — uses `send_streaming` for S3 backends (richer headers from backend), - /// falls back to `object_store` for non-S3 backends. - async fn handle_head( - &self, - config: &BucketConfig, - key: &str, - headers: &HeaderMap, - ) -> Result, ProxyError> { - if config.supports_s3_multipart() { - return self.handle_head_s3(config, key, headers).await; - } - self.handle_head_object_store(config, key).await - } - - /// HEAD for S3 backends via `send_streaming`. - async fn handle_head_s3( - &self, - config: &BucketConfig, - key: &str, - headers: &HeaderMap, - ) -> Result, ProxyError> { - let operation = S3Operation::HeadObject { - bucket: String::new(), - key: key.to_string(), - }; - let backend_url = build_backend_url(config, &operation)?; - - let mut req_headers = HeaderMap::new(); - - // Forward conditional headers - for header_name in &[ - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", - ] { - if let Some(val) = headers.get(*header_name) { - req_headers.insert(*header_name, val.clone()); + let mut fwd_headers = HeaderMap::new(); + for name in forward_header_names { + if let Some(v) = original_headers.get(*name) { + fwd_headers.insert(*name, v.clone()); } } - sign_s3_request(&Method::HEAD, &backend_url, &mut req_headers, config, UNSIGNED_PAYLOAD)?; - - tracing::debug!(backend_url = %backend_url, "HEAD via send_streaming (S3)"); - - let raw = self.backend.send_streaming(Method::HEAD, backend_url, req_headers).await?; - - // Forward response headers from the backend - let mut resp_headers = HeaderMap::new(); - for header_name in STREAMING_RESPONSE_HEADERS { - if let Some(val) = raw.headers.get(*header_name) { - resp_headers.insert(*header_name, val.clone()); - } - } - - Ok(ProxyResult { - status: raw.status, - headers: resp_headers, - body: ProxyResponseBody::Empty, - }) - } - - /// HEAD for non-S3 backends via `object_store`. - async fn handle_head_object_store( - &self, - config: &BucketConfig, - key: &str, - ) -> Result, ProxyError> { - let store = self.backend.create_store(config)?; - let path = build_object_path(config, key); - - tracing::debug!(path = %path, "HEAD via object_store"); - - let meta = store - .head(&path) - .await - .map_err(ProxyError::from_object_store_error)?; - - let mut resp_headers = HeaderMap::new(); - if let Some(etag) = &meta.e_tag { - resp_headers.insert("etag", etag.parse().unwrap()); - } - resp_headers.insert( - "last-modified", - meta.last_modified - .format("%a, %d %b %Y %H:%M:%S GMT") - .to_string() - .parse() - .unwrap(), - ); - resp_headers.insert("content-length", meta.size.to_string().parse().unwrap()); - resp_headers.insert("accept-ranges", "bytes".parse().unwrap()); - - Ok(ProxyResult { - status: 200, - headers: resp_headers, - body: ProxyResponseBody::Empty, - }) - } - - /// PUT via object_store - async fn handle_put( - &self, - config: &BucketConfig, - key: &str, - body: Bytes, - ) -> Result, ProxyError> { - let store = self.backend.create_store(config)?; - let path = build_object_path(config, key); - - tracing::debug!(path = %path, body_len = body.len(), "PUT via object_store"); - - let payload = PutPayload::from(body); - let result = store - .put(&path, payload) - .await - .map_err(ProxyError::from_object_store_error)?; - - let mut resp_headers = HeaderMap::new(); - if let Some(etag) = &result.e_tag { - resp_headers.insert("etag", etag.parse().unwrap()); - } - - Ok(ProxyResult { - status: 200, - headers: resp_headers, - body: ProxyResponseBody::Empty, - }) - } - - /// DELETE via object_store - async fn handle_delete( - &self, - config: &BucketConfig, - key: &str, - ) -> Result, ProxyError> { - let store = self.backend.create_store(config)?; - let path = build_object_path(config, key); - - tracing::debug!(path = %path, "DELETE via object_store"); - - store - .delete(&path) - .await - .map_err(ProxyError::from_object_store_error)?; - - Ok(ProxyResult { - status: 204, - headers: HeaderMap::new(), - body: ProxyResponseBody::Empty, + Ok(ForwardRequest { + method, + url, + headers: fwd_headers, }) } @@ -487,7 +338,7 @@ where config: &BucketConfig, raw_query: Option<&str>, list_rewrite: Option<&ListRewrite>, - ) -> Result, ProxyError> { + ) -> Result { let store = self.backend.create_store(config)?; // Extract prefix from query string @@ -549,16 +400,13 @@ where }) } - /// Multipart operations via raw signed HTTP - async fn handle_multipart( + /// Execute a multipart operation via raw signed HTTP. + async fn execute_multipart( &self, - method: &Method, - operation: &S3Operation, - bucket_config: &BucketConfig, - original_headers: &HeaderMap, + pending: &PendingRequest, body: Bytes, - ) -> Result, ProxyError> { - let backend_url = build_backend_url(bucket_config, operation)?; + ) -> Result { + let backend_url = build_backend_url(&pending.bucket_config, &pending.operation)?; tracing::debug!(backend_url = %backend_url, "multipart via raw HTTP"); @@ -570,7 +418,7 @@ where "content-length", "content-md5", ] { - if let Some(val) = original_headers.get(*header_name) { + if let Some(val) = pending.original_headers.get(*header_name) { headers.insert(*header_name, val.clone()); } } @@ -581,11 +429,11 @@ where hash_payload(&body) }; - sign_s3_request(method, &backend_url, &mut headers, bucket_config, &payload_hash)?; + sign_s3_request(&pending.method, &backend_url, &mut headers, &pending.bucket_config, &payload_hash)?; let raw_resp = self .backend - .send_raw(method.clone(), backend_url, headers, body) + .send_raw(pending.method.clone(), backend_url, headers, body) .await?; tracing::debug!(status = raw_resp.status, "multipart backend response"); @@ -598,15 +446,15 @@ where } } -/// The result of handling a proxy request, generic over the native body type. -pub struct ProxyResult { +/// The result of handling a proxy request. +pub struct ProxyResult { pub status: u16, pub headers: HeaderMap, - pub body: ProxyResponseBody, + pub body: ProxyResponseBody, } -/// Headers to forward from backend streaming responses. -const STREAMING_RESPONSE_HEADERS: &[&str] = &[ +/// Headers to forward from backend responses (used by runtimes for Forward responses). +pub const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ "content-type", "content-length", "content-range", @@ -616,9 +464,12 @@ const STREAMING_RESPONSE_HEADERS: &[&str] = &[ "content-encoding", "content-disposition", "cache-control", + "x-amz-request-id", + "x-amz-version-id", + "location", ]; -fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { +fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { let xml = ErrorResponse::from_proxy_error(err, resource, request_id).to_xml(); let body = ProxyResponseBody::from_bytes(Bytes::from(xml)); let mut headers = HeaderMap::new(); @@ -633,8 +484,7 @@ fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> Prox /// Sign an outbound S3 request using credentials from the bucket config. /// -/// If credentials are configured (`access_key_id` + `secret_access_key`), -/// applies SigV4 signing. Otherwise, just sets the Host header. +/// Used for multipart operations only. CRUD operations use presigned URLs. fn sign_s3_request( method: &Method, url: &str, @@ -790,30 +640,9 @@ fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite> key } -/// Parse an HTTP Range header value into an object_store GetRange. -fn parse_range_header(value: &str) -> Option { - let range_str = value.strip_prefix("bytes=")?; - - if let Some(suffix) = range_str.strip_prefix('-') { - // bytes=-N (suffix) - let n: u64 = suffix.parse().ok()?; - return Some(GetRange::Suffix(n)); - } - - let (start_str, end_str) = range_str.split_once('-')?; - let start: u64 = start_str.parse().ok()?; - - if end_str.is_empty() { - // bytes=N- (offset to end) - Some(GetRange::Offset(start)) - } else { - // bytes=N-M (bounded, HTTP is inclusive, object_store is exclusive) - let end: u64 = end_str.parse().ok()?; - Some(GetRange::Bounded(start..end + 1)) - } -} - /// Build the backend URL for an S3 operation. +/// +/// Used for multipart operations that go through raw signed HTTP. pub fn build_backend_url( config: &BucketConfig, operation: &S3Operation, diff --git a/crates/libs/core/src/response_body.rs b/crates/libs/core/src/response_body.rs index 6aaea0d..48b75a2 100644 --- a/crates/libs/core/src/response_body.rs +++ b/crates/libs/core/src/response_body.rs @@ -1,32 +1,24 @@ //! Response body type for the proxy. //! -//! [`ProxyResponseBody`] is generic over a native body type `N` so that each -//! runtime can carry its platform-native streaming body through the handler -//! without intermediate conversion (e.g. JS ReadableStream on CF Workers). +//! [`ProxyResponseBody`] carries non-streaming response data. Streaming +//! responses (GET, PUT) are handled by the runtime via [`ForwardRequest`] +//! presigned URLs — the handler never touches those bytes. use bytes::Bytes; -use futures::stream::BoxStream; /// The body of a proxy response. /// -/// Generic over `N`, the runtime's native streaming body type. -/// The default `N = ()` is used for responses that don't carry a native body -/// (errors, list XML, HEAD, etc.). -pub enum ProxyResponseBody { - /// Streaming response from `object_store` GET (non-S3 backends). - /// Bytes arrive lazily in chunks. - Stream(BoxStream<'static, Result>), +/// Only used for responses the handler constructs directly (errors, LIST XML, +/// multipart XML, HEAD metadata). Streaming GET/PUT bodies bypass this type +/// entirely via the `Forward` action. +pub enum ProxyResponseBody { /// Fixed bytes (error XML, list XML, multipart XML responses, etc.). Bytes(Bytes), /// Empty body (HEAD responses, etc.). Empty, - /// Runtime-native streaming body, bypassing Rust stream intermediaries. - /// On CF Workers this is `Option`; - /// on the server runtime it's a reqwest byte stream. - Native(N), } -impl ProxyResponseBody { +impl ProxyResponseBody { /// Create a response body from raw bytes. pub fn from_bytes(bytes: Bytes) -> Self { if bytes.is_empty() { diff --git a/crates/runtimes/cf-workers/README.md b/crates/runtimes/cf-workers/README.md index 086af3f..dbd0d94 100644 --- a/crates/runtimes/cf-workers/README.md +++ b/crates/runtimes/cf-workers/README.md @@ -1,6 +1,6 @@ # s3-proxy-cf-workers -Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, using `object_store` with a custom `FetchConnector` that bridges to the Workers Fetch API. +Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, using presigned URLs for zero-copy streaming and `object_store` with a custom `FetchConnector` for LIST operations. ## How It Works @@ -11,23 +11,23 @@ Client request -> Pick resolver: - SOURCE_API_URL set? -> SourceCoopResolver (dynamic Source Cooperative backends) - Otherwise -> DefaultResolver (static PROXY_CONFIG) - -> ProxyHandler::handle_request() (from s3-proxy-core) - -> object_store (via FetchConnector) or raw fetch for multipart - -> ProxyResponseBody converted to worker::Response + -> ProxyHandler::resolve_request() (from s3-proxy-core) + -> Forward: fetch(presigned URL) with ReadableStream passthrough (GET/HEAD/PUT/DELETE) + -> Response: LIST XML via object_store, errors, synthetic responses + -> NeedsBody: multipart operations via raw signed HTTP ``` -`WorkerBackend` implements `ProxyBackend` using a custom `FetchConnector` that bridges `object_store` HTTP requests to the Workers Fetch API. Since JS interop types are `!Send`, `spawn_local` + channel patterns are used to bridge to the `Send` context that `object_store` expects. Response body streams are converted to JS `ReadableStream` via `TransformStream` for delivery to clients. +`WorkerBackend` implements `ProxyBackend` with three capabilities: `create_signer()` generates presigned URLs for CRUD operations (executed via the Fetch API with JS `ReadableStream` passthrough — zero Rust stream involvement), `create_store()` uses a custom `FetchConnector` for LIST operations, and `send_raw()` handles multipart uploads. `FetchConnector` bridges `object_store` to the Workers Fetch API using `spawn_local` + channel patterns (since JS interop types are `!Send`). ## Module Overview ``` src/ -├── lib.rs Worker entry point, request/response conversion (thin adapter) -├── body.rs ProxyResponseBody → worker::Response conversion -├── client.rs WorkerBackend implementing ProxyBackend, fetch_json helper -├── fetch_connector.rs FetchConnector/FetchService bridging object_store to Fetch API -├── source_api.rs HTTP client for the Source Cooperative API -└── source_resolver.rs SourceCoopResolver implementing RequestResolver +├── lib.rs Worker entry point, two-phase request handling, Forward execution +├── body.rs ProxyResult → worker::Response conversion (Bytes/Empty only) +├── client.rs WorkerBackend implementing ProxyBackend, WorkerHttpClient +├── fetch_connector.rs FetchConnector/FetchService bridging object_store to Fetch API (LIST only) +└── tracing_layer.rs Minimal tracing subscriber for Workers console_log ``` ## Operating Modes @@ -95,8 +95,7 @@ Then add a branch in `lib.rs`: if let Ok(my_config) = env.var("MY_MODE") { let resolver = MyResolver::new(/* ... */); let handler = ProxyHandler::new(client::WorkerBackend, resolver); - let result = handler.handle_request(method, &path, query.as_deref(), &headers, body).await; - return build_worker_response(result); + return handle_action(&req, method, &handler, &path, query.as_deref(), &headers).await; } ``` diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs index 1700cf7..0a0855c 100644 --- a/crates/runtimes/cf-workers/src/body.rs +++ b/crates/runtimes/cf-workers/src/body.rs @@ -1,21 +1,19 @@ //! Response body conversion for the Cloudflare Workers runtime. //! -//! Converts [`ProxyResponseBody`] to `worker::Response`. +//! Converts [`ProxyResult`] to `worker::Response`. Only handles non-streaming +//! bodies (Bytes, Empty). Streaming responses go through the Forward path +//! in `lib.rs`, which uses the Fetch API directly. -use futures::StreamExt; -use js_sys::Uint8Array; use s3_proxy_core::proxy::ProxyResult; use s3_proxy_core::response_body::ProxyResponseBody; -use wasm_bindgen_futures::spawn_local; use worker::{Headers, Response}; /// Build a `worker::Response` from a `ProxyResult`. /// -/// - `Native` bodies (JS `ReadableStream`) are passed through directly — zero conversion. -/// - `Stream` bodies (Rust `BoxStream`) are bridged to JS via a `TransformStream`. -/// - `Bytes`/`Empty` are returned as fixed responses. +/// Only handles `Bytes` and `Empty` bodies (LIST XML, errors, multipart XML). +/// Streaming Forward responses are built directly in `lib.rs`. pub fn build_worker_response( - result: ProxyResult>, + result: ProxyResult, ) -> Result { let resp_headers = Headers::new(); for (key, value) in result.headers.iter() { @@ -25,82 +23,6 @@ pub fn build_worker_response( } match result.body { - ProxyResponseBody::Native(maybe_stream) => { - // Pass the JS ReadableStream directly — no Rust stream conversion! - let ws_headers = web_sys::Headers::new() - .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } - } - - let init = web_sys::ResponseInit::new(); - init.set_status(result.status); - init.set_headers(&ws_headers.into()); - - let ws_response = web_sys::Response::new_with_opt_readable_stream_and_init( - maybe_stream.as_ref(), - &init, - ) - .map_err(|e| { - worker::Error::RustError(format!("failed to build response: {:?}", e)) - })?; - - Ok(ws_response.into()) - } - ProxyResponseBody::Stream(stream) => { - // Bridge Rust Stream -> JS ReadableStream via TransformStream - let transform = web_sys::TransformStream::new() - .map_err(|e| worker::Error::RustError(format!("TransformStream error: {:?}", e)))?; - - let writable = transform.writable(); - let readable = transform.readable(); - - // Spawn a task to pump chunks from the Rust stream into the JS writable side - spawn_local(async move { - let writer = writable.get_writer().unwrap(); - let mut stream = stream; - - while let Some(chunk_result) = stream.next().await { - match chunk_result { - Ok(bytes) => { - let uint8 = Uint8Array::from(bytes.as_ref()); - if let Err(_) = wasm_bindgen_futures::JsFuture::from( - writer.write_with_chunk(&uint8.into()), - ) - .await - { - break; - } - } - Err(_) => break, - } - } - let _ = wasm_bindgen_futures::JsFuture::from(writer.close()).await; - }); - - // Build the response from the readable side - let ws_headers = web_sys::Headers::new() - .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } - } - - let init = web_sys::ResponseInit::new(); - init.set_status(result.status); - init.set_headers(&ws_headers.into()); - - let ws_response = - web_sys::Response::new_with_opt_readable_stream_and_init(Some(&readable), &init) - .map_err(|e| { - worker::Error::RustError(format!("failed to build response: {:?}", e)) - })?; - - Ok(ws_response.into()) - } ProxyResponseBody::Bytes(b) => Ok(Response::from_bytes(b.to_vec())? .with_status(result.status) .with_headers(resp_headers)), diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 10c98b2..6dae170 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -7,10 +7,9 @@ use crate::fetch_connector::FetchConnector; use bytes::Bytes; use http::HeaderMap; +use object_store::signer::Signer; use object_store::ObjectStore; -use s3_proxy_core::backend::{ - build_object_store, ProxyBackend, RawResponse, RawStreamingResponse, StoreBuilder, -}; +use s3_proxy_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; use s3_proxy_source_coop::api::{CacheOptions, HttpClient}; @@ -111,35 +110,6 @@ impl HttpClient for WorkerHttpClient { } } -/// Headers to extract from backend streaming responses. -const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ - "content-type", - "content-length", - "content-range", - "etag", - "last-modified", - "accept-ranges", - "content-encoding", - "content-disposition", - "cache-control", - "x-amz-request-id", - "x-amz-version-id", - "location", -]; - -/// Extract response headers from a `web_sys::Headers` using an allowlist. -fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { - let mut resp_headers = HeaderMap::new(); - for name in RESPONSE_HEADER_ALLOWLIST { - if let Ok(Some(value)) = ws_headers.get(name) { - if let Ok(parsed) = value.parse() { - resp_headers.insert(*name, parsed); - } - } - } - resp_headers -} - /// Backend for the Cloudflare Workers runtime. /// /// Uses `FetchConnector` for `object_store` HTTP requests and `web_sys::fetch` @@ -148,14 +118,16 @@ fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { pub struct WorkerBackend; impl ProxyBackend for WorkerBackend { - type NativeBody = Option; - fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { build_object_store(config, |b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), }) } + fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { + build_signer(config) + } + async fn send_raw( &self, method: http::Method, @@ -219,58 +191,33 @@ impl ProxyBackend for WorkerBackend { body: Bytes::from(resp_bytes), }) } +} - async fn send_streaming( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - ) -> Result, ProxyError> { - tracing::debug!( - method = %method, - url = %url, - "worker: sending streaming backend request via Fetch API" - ); - - // Build web_sys::Headers - let ws_headers = web_sys::Headers::new() - .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; +/// Headers to extract from backend responses. +pub const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ + "content-type", + "content-length", + "content-range", + "etag", + "last-modified", + "accept-ranges", + "content-encoding", + "content-disposition", + "cache-control", + "x-amz-request-id", + "x-amz-version-id", + "location", +]; - for (key, value) in headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); +/// Extract response headers from a `web_sys::Headers` using an allowlist. +pub fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { + let mut resp_headers = HeaderMap::new(); + for name in RESPONSE_HEADER_ALLOWLIST { + if let Ok(Some(value)) = ws_headers.get(name) { + if let Ok(parsed) = value.parse() { + resp_headers.insert(*name, parsed); } } - - // Build web_sys::RequestInit - let init = web_sys::RequestInit::new(); - init.set_method(method.as_str()); - init.set_headers(&ws_headers.into()); - - let ws_request = - web_sys::Request::new_with_str_and_init(&url, &init).map_err(|e| { - ProxyError::BackendError(format!("failed to create request: {:?}", e)) - })?; - - // Fetch via worker — returns a JS Response with a ReadableStream body - let worker_req: worker::Request = ws_request.into(); - let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { - ProxyError::BackendError(format!("fetch failed: {}", e)) - })?; - - let status = worker_resp.status_code(); - - // Extract response headers and body stream from the web_sys::Response - let ws_response: web_sys::Response = worker_resp.into(); - let resp_headers = extract_response_headers(&ws_response.headers()); - - // Get the ReadableStream directly — no conversion to Rust types! - let body = ws_response.body(); - - Ok(RawStreamingResponse { - status, - headers: resp_headers, - body, - }) } + resp_headers } diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index bee9bec..480b80c 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -1,17 +1,20 @@ //! Cloudflare Workers runtime for the S3 proxy gateway. //! //! This crate provides implementations of core traits using Cloudflare Workers -//! primitives. Response bodies from `object_store` are bridged from Rust -//! `Stream` to JS `ReadableStream` via a `TransformStream`. +//! primitives. Uses a two-phase request handling model: +//! +//! 1. **`resolve_request`** determines the action (Forward, Response, NeedsBody) +//! 2. **Forward** requests execute presigned URLs via the Fetch API, passing +//! JS `ReadableStream` bodies directly — zero Rust stream involvement. //! //! # Architecture //! //! ```text //! Client -> Worker (JS Request) //! -> resolve request (core resolver or Source Cooperative resolver) -//! -> object_store operation (via FetchConnector -> Fetch API) -//! -> ProxyResponseBody::Stream -> TransformStream -> JS ReadableStream -//! -> return JS Response +//! -> Forward: fetch(presigned URL) with ReadableStream passthrough +//! -> Response: LIST XML via object_store, errors, synthetic responses +//! -> NeedsBody: multipart operations via raw signed HTTP //! ``` //! //! # Configuration @@ -28,9 +31,10 @@ mod fetch_connector; mod tracing_layer; use body::build_worker_response; +use client::{extract_response_headers, WorkerBackend}; use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; -use s3_proxy_core::proxy::ProxyHandler; -use s3_proxy_core::resolver::DefaultResolver; +use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler}; +use s3_proxy_core::resolver::{DefaultResolver, RequestResolver}; use s3_proxy_source_coop::api::{CacheTtls, SourceApiClient}; use s3_proxy_source_coop::resolver::SourceCoopResolver; use worker::*; @@ -55,13 +59,6 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let query = url.query().map(|q| q.to_string()); let headers = convert_headers(&req); - // Materialize request body to Bytes for PUT/POST, empty for others - let body = if matches!(method, http::Method::PUT | http::Method::POST) { - read_request_body(&req).await? - } else { - bytes::Bytes::new() - }; - // Source Cooperative API mode: when SOURCE_API_URL is set, resolve backends // dynamically from the Source API instead of static PROXY_CONFIG. if let Ok(source_api_url) = env.var("SOURCE_API_URL") { @@ -109,13 +106,9 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { cache_ttls, ); let resolver = SourceCoopResolver::new(api_client); - let handler = ProxyHandler::new(client::WorkerBackend, resolver); + let handler = ProxyHandler::new(WorkerBackend, resolver); - let result = handler - .handle_request(method, &path, query.as_deref(), &headers, body) - .await; - - return build_worker_response(result); + return handle_action(&req, method, &handler, &path, query.as_deref(), &headers).await; } // Load PROXY_CONFIG from environment. @@ -140,13 +133,104 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); let resolver = DefaultResolver::new(config, virtual_host_domain); - let handler = ProxyHandler::new(client::WorkerBackend, resolver); + let handler = ProxyHandler::new(WorkerBackend, resolver); + + handle_action(&req, method, &handler, &path, query.as_deref(), &headers).await +} + +// ── Two-phase request handling ────────────────────────────────────── - let result = handler - .handle_request(method, &path, query.as_deref(), &headers, body) +/// Handle the resolved action for any resolver type. +async fn handle_action( + req: &Request, + method: http::Method, + handler: &ProxyHandler, + path: &str, + query: Option<&str>, + headers: &http::HeaderMap, +) -> Result { + let action = handler + .resolve_request(method, path, query, headers) .await; - build_worker_response(result) + match action { + HandlerAction::Response(result) => build_worker_response(result), + HandlerAction::Forward(fwd) => execute_forward(req, fwd).await, + HandlerAction::NeedsBody(pending) => { + let body = read_request_body(req).await?; + let result = handler.handle_with_body(pending, body).await; + build_worker_response(result) + } + } +} + +/// Execute a Forward request via the Fetch API. +/// +/// For PUT: passes the original JS `ReadableStream` body directly to fetch. +/// For GET: returns the response `ReadableStream` directly to the client. +/// Zero Rust stream involvement — bytes never cross the WASM boundary. +async fn execute_forward( + req: &Request, + fwd: ForwardRequest, +) -> Result { + // Build request headers + let ws_headers = web_sys::Headers::new() + .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; + + for (key, value) in fwd.headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_headers.set(key.as_str(), v); + } + } + + // Build fetch request + let init = web_sys::RequestInit::new(); + init.set_method(fwd.method.as_str()); + init.set_headers(&ws_headers.into()); + + // For PUT: pass original request body stream directly + if fwd.method == http::Method::PUT { + if let Some(body_stream) = req.inner().body() { + init.set_body(&body_stream.into()); + } + } + + let ws_request = + web_sys::Request::new_with_str_and_init(fwd.url.as_str(), &init) + .map_err(|e| worker::Error::RustError(format!("request error: {:?}", e)))?; + + // Execute fetch + let worker_req: worker::Request = ws_request.into(); + let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { + worker::Error::RustError(format!("forward fetch failed: {}", e)) + })?; + + let status = worker_resp.status_code(); + let ws_response: web_sys::Response = worker_resp.into(); + + // Forward allowlisted response headers + let resp_headers = extract_response_headers(&ws_response.headers()); + let ws_resp_headers = web_sys::Headers::new() + .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; + for (key, value) in resp_headers.iter() { + if let Ok(v) = value.to_str() { + let _ = ws_resp_headers.set(key.as_str(), v); + } + } + + // Build response with the backend's ReadableStream body (passthrough) + let resp_init = web_sys::ResponseInit::new(); + resp_init.set_status(status); + resp_init.set_headers(&ws_resp_headers.into()); + + let body = ws_response.body(); + let response = web_sys::Response::new_with_opt_readable_stream_and_init( + body.as_ref(), + &resp_init, + ) + .map_err(|e| worker::Error::RustError(format!("response error: {:?}", e)))?; + + Ok(response.into()) } // ── Shared helpers ────────────────────────────────────────────────── diff --git a/crates/runtimes/server/README.md b/crates/runtimes/server/README.md index 0c9dad5..4242a8d 100644 --- a/crates/runtimes/server/README.md +++ b/crates/runtimes/server/README.md @@ -6,7 +6,7 @@ Tokio/Hyper runtime for the S3 proxy gateway. This is the container-deployment c A `ProxyBackend` implementation plus a server binary: -**`ServerBackend`** — implements `ProxyBackend`. Uses `object_store` with its default HTTP connector for high-level operations (GET, HEAD, PUT, LIST) and `reqwest` for raw multipart requests. GET responses stream from `object_store` through Hyper without buffering. +**`ServerBackend`** — implements `ProxyBackend`. Provides `create_signer()` for presigned URL generation (GET/HEAD/PUT/DELETE), `create_store()` for LIST operations, and `send_raw()` via reqwest for multipart uploads. All Forward operations (GET/HEAD/PUT/DELETE) execute presigned URLs via reqwest; GET response bodies and PUT request bodies stream without buffering. **`server::run()`** — starts a Hyper HTTP server that accepts connections and delegates to `ProxyHandler` with a `DefaultResolver`. Supports both path-style (`/bucket/key`) and virtual-hosted-style (`bucket.s3.example.com/key`) routing via the resolver's `virtual_host_domain` setting. @@ -15,9 +15,9 @@ A `ProxyBackend` implementation plus a server binary: ``` src/ ├── lib.rs Crate root -├── body.rs ProxyResponseBody → Hyper streaming response conversion +├── body.rs ProxyResult → Hyper response conversion (Bytes/Empty only) ├── client.rs ServerBackend implementing ProxyBackend -├── server.rs Hyper server setup, request routing +├── server.rs Hyper server setup, two-phase request handling, Forward execution └── bin/ └── s3-proxy.rs CLI binary entry point ``` @@ -99,17 +99,19 @@ impl RequestResolver for MyResolver { let backend = ServerBackend::new(); let handler = ProxyHandler::new(backend, MyResolver::new()); -// Use handler.handle_request() in your Hyper service. +// Use handler.resolve_request() in your Hyper service — returns HandlerAction. ``` -See `s3-proxy-cf-workers/src/source_resolver.rs` for a complete example. +See `crates/libs/source-coop/src/resolver.rs` for a complete example. ## Streaming Behavior -For **GET** responses, `object_store` returns a `BoxStream` which is bridged to a Hyper streaming response body. Bytes flow through without buffering the entire object in memory. +For **GET** responses, the handler generates a presigned URL and returns a `Forward` action. The server executes the URL via reqwest and streams the response body through Hyper using `bytes_stream()` — no buffering. -For **HEAD** responses, only metadata is returned (empty body). +For **PUT** requests, the handler generates a presigned URL and returns a `Forward` action. The server streams the Hyper `Incoming` body directly to the presigned URL via `reqwest::Body::wrap_stream()` — no body materialization. -For **PUT** request bodies, the incoming Hyper body is collected to `Bytes` before passing to `object_store::put()`. +For **HEAD/DELETE** responses, the handler generates a presigned URL. The server executes it and returns the status + headers. -For **multipart uploads**, operations are sent as raw signed HTTP requests via `reqwest`. The `CompleteMultipartUpload` request body (a small XML manifest) is the only body the proxy fully reads and parses. +For **LIST** responses, `object_store` handles the request internally and the handler returns a `Response` with XML body. + +For **multipart uploads**, operations are sent as raw signed HTTP requests via `reqwest`. The request body is materialized to `Bytes` first (multipart XML payloads are small). diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs index bc552ca..671384a 100644 --- a/crates/runtimes/server/src/body.rs +++ b/crates/runtimes/server/src/body.rs @@ -1,9 +1,10 @@ //! Response body conversion for the server runtime. //! -//! Converts [`ProxyResponseBody`] to a streaming hyper response body. +//! Converts [`ProxyResult`] to a hyper response body. Streaming responses +//! (from Forward requests) are handled directly in `server.rs`. use bytes::Bytes; -use futures::{Stream, TryStreamExt}; +use futures::Stream; use http::Response; use http_body_util::{Either, Empty, Full, StreamBody}; use hyper::body::Frame; @@ -11,22 +12,22 @@ use s3_proxy_core::proxy::ProxyResult; use s3_proxy_core::response_body::ProxyResponseBody; use std::pin::Pin; -/// The server's native body type from `send_streaming`. -type NativeStream = Pin> + Send>>; - /// A boxed streaming body type that erases concrete stream types. type BoxedStreamBody = StreamBody< - std::pin::Pin< - Box, std::io::Error>> + Send>, + Pin< + Box, std::io::Error>> + Send>, >, >; -/// The server response body type: either a stream, fixed bytes, or empty. +/// The server response body type: either a stream (Forward) or fixed bytes/empty (Response). pub type ServerResponseBody = Either, Empty>>; -/// Convert a `ProxyResult` to a hyper `Response` with a streaming body. +/// Convert a `ProxyResult` to a hyper `Response`. +/// +/// Only handles `Bytes` and `Empty` bodies (LIST, errors, multipart responses). +/// Streaming Forward responses are built directly in `server.rs`. pub fn build_hyper_response( - result: ProxyResult, + result: ProxyResult, ) -> Result, Box> { let mut builder = Response::builder().status(result.status); @@ -35,18 +36,6 @@ pub fn build_hyper_response( } let body = match result.body { - ProxyResponseBody::Native(stream) => { - let framed = stream - .map_ok(Frame::data) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); - Either::Left(StreamBody::new(Box::pin(framed) as std::pin::Pin, std::io::Error>> + Send>>)) - } - ProxyResponseBody::Stream(stream) => { - let framed = stream - .map_ok(Frame::data) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); - Either::Left(StreamBody::new(Box::pin(framed) as std::pin::Pin, std::io::Error>> + Send>>)) - } ProxyResponseBody::Bytes(b) => Either::Right(Either::Left(Full::new(b))), ProxyResponseBody::Empty => Either::Right(Either::Right(Empty::new())), }; diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index fd175bc..3d5ad4e 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -1,13 +1,12 @@ //! Server backend using reqwest for raw HTTP and default object_store connector. use bytes::Bytes; -use futures::Stream; use http::HeaderMap; +use object_store::signer::Signer; use object_store::ObjectStore; -use s3_proxy_core::backend::{build_object_store, ProxyBackend, RawResponse, RawStreamingResponse}; +use s3_proxy_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; -use std::pin::Pin; use std::sync::Arc; /// Backend for the Tokio/Hyper server runtime. @@ -28,6 +27,11 @@ impl ServerBackend { .expect("failed to build reqwest client"), } } + + /// Access the underlying reqwest client for Forward request execution. + pub fn client(&self) -> &reqwest::Client { + &self.client + } } impl Default for ServerBackend { @@ -37,12 +41,14 @@ impl Default for ServerBackend { } impl ProxyBackend for ServerBackend { - type NativeBody = Pin> + Send>>; - fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { build_object_store(config, |b| b) } + fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { + build_signer(config) + } + async fn send_raw( &self, method: http::Method, @@ -83,38 +89,4 @@ impl ProxyBackend for ServerBackend { body: resp_body, }) } - - async fn send_streaming( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - ) -> Result, ProxyError> { - tracing::debug!( - method = %method, - url = %url, - "server: sending streaming backend request via reqwest" - ); - - let mut req_builder = self.client.request(method, &url); - - for (key, value) in headers.iter() { - req_builder = req_builder.header(key, value); - } - - let response = req_builder.send().await.map_err(|e| { - tracing::error!(error = %e, "reqwest streaming request failed"); - ProxyError::BackendError(e.to_string()) - })?; - - let status = response.status().as_u16(); - let resp_headers = response.headers().clone(); - let body = Box::pin(response.bytes_stream()); - - Ok(RawStreamingResponse { - status, - headers: resp_headers, - body, - }) - } } diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 70c0c29..9d9ce50 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -3,15 +3,17 @@ use crate::body::{build_hyper_response, ServerResponseBody}; use crate::client::ServerBackend; use bytes::Bytes; -use http::{Request, Response}; -use http_body_util::{BodyExt, Either, Full}; -use hyper::body::Incoming; +use futures::{Stream, TryStreamExt}; +use http::{HeaderMap, Response}; +use http_body_util::{BodyExt, BodyStream, Either, Full, StreamBody}; +use hyper::body::{Frame, Incoming}; use hyper::service::service_fn; use hyper_util::rt::{TokioExecutor, TokioIo}; use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::proxy::ProxyHandler; +use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; use s3_proxy_core::resolver::DefaultResolver; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::Arc; use tokio::net::TcpListener; @@ -55,6 +57,7 @@ where P: ConfigProvider + Send + Sync + 'static, { let backend = ServerBackend::new(); + let reqwest_client = backend.client().clone(); let resolver = DefaultResolver::new(config, server_config.virtual_host_domain); let handler = Arc::new(ProxyHandler::new(backend, resolver)); @@ -64,10 +67,12 @@ where loop { let (stream, remote_addr) = listener.accept().await?; let handler = handler.clone(); + let client = reqwest_client.clone(); tokio::spawn(async move { - let service = service_fn(move |req: Request| { + let service = service_fn(move |req: http::Request| { let handler = handler.clone(); + let client = client.clone(); async move { tracing::debug!( @@ -76,7 +81,7 @@ where uri = %req.uri(), "incoming connection" ); - let result = handle_hyper_request(req, &handler).await; + let result = handle_hyper_request(req, &handler, &client).await; match result { Ok(resp) => Ok::<_, hyper::Error>(resp), Err(e) => { @@ -102,24 +107,86 @@ where } async fn handle_hyper_request( - req: Request, + req: http::Request, handler: &ProxyHandler, + client: &reqwest::Client, ) -> Result, Box> where R: s3_proxy_core::resolver::RequestResolver + Send + Sync, { - let method = req.method().clone(); - let uri = req.uri().clone(); - let path = uri.path(); - let query = uri.query(); - let headers = req.headers().clone(); + let (parts, incoming_body) = req.into_parts(); + let method = parts.method; + let uri = parts.uri; + let path = uri.path().to_string(); + let query = uri.query().map(|q| q.to_string()); + let headers = parts.headers; - // Materialize incoming body to Bytes - let incoming_bytes = req.into_body().collect().await?.to_bytes(); - - let result = handler - .handle_request(method, path, query, &headers, incoming_bytes) + let action = handler + .resolve_request(method, &path, query.as_deref(), &headers) .await; - build_hyper_response(result) + match action { + HandlerAction::Response(result) => build_hyper_response(result), + HandlerAction::Forward(fwd) => { + forward_to_backend(client, fwd, incoming_body).await + } + HandlerAction::NeedsBody(pending) => { + let body = incoming_body.collect().await?.to_bytes(); + let result = handler.handle_with_body(pending, body).await; + build_hyper_response(result) + } + } +} + +/// Execute a Forward request via reqwest, streaming both request and response bodies. +async fn forward_to_backend( + client: &reqwest::Client, + fwd: ForwardRequest, + incoming_body: Incoming, +) -> Result, Box> { + let mut req_builder = client.request(fwd.method.clone(), fwd.url.as_str()); + + for (k, v) in fwd.headers.iter() { + req_builder = req_builder.header(k, v); + } + + // Attach streaming body for PUT + if fwd.method == http::Method::PUT { + let body_stream = BodyStream::new(incoming_body) + .try_filter_map(|frame| async move { + Ok(frame.into_data().ok()) + }); + req_builder = req_builder.body(reqwest::Body::wrap_stream(body_stream)); + } + + let backend_resp = req_builder.send().await.map_err(|e| { + tracing::error!(error = %e, "forward request failed"); + Box::new(e) as Box + })?; + + let status = backend_resp.status().as_u16(); + + // Forward allowlisted response headers + let mut resp_headers = HeaderMap::new(); + for name in RESPONSE_HEADER_ALLOWLIST { + if let Some(v) = backend_resp.headers().get(*name) { + resp_headers.insert(*name, v.clone()); + } + } + + // Stream the response body + let body_stream = backend_resp.bytes_stream(); + let framed = body_stream + .map_ok(Frame::data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); + let body: ServerResponseBody = Either::Left(StreamBody::new( + Box::pin(framed) as Pin, std::io::Error>> + Send>> + )); + + let mut builder = Response::builder().status(status); + for (k, v) in resp_headers.iter() { + builder = builder.header(k, v); + } + + Ok(builder.body(body)?) } From 5bbe04926a3dc5da243c93f6334dfb93adb178ba Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 16:21:27 -0500 Subject: [PATCH 17/82] chore: add makefile for ease of use --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f15b63d --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: check test run\:server run\:workers + +check: + cargo check + cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown + +test: + cargo test + +run\:server: + cargo run -p s3-proxy-server -- $(ARGS) + +run\:workers: + npx wrangler dev --cwd crates/runtimes/cf-workers From 54f7ea175dc2856e8ae7b6356ad7e19afb72028b Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 21:47:11 -0500 Subject: [PATCH 18/82] fix: update API response structure to list products instead of repositories --- crates/libs/source-coop/src/api.rs | 10 +++++----- crates/libs/source-coop/src/resolver.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/libs/source-coop/src/api.rs b/crates/libs/source-coop/src/api.rs index 326df35..978612f 100644 --- a/crates/libs/source-coop/src/api.rs +++ b/crates/libs/source-coop/src/api.rs @@ -133,16 +133,16 @@ pub struct PermissionsResponse { pub write: bool, } -/// API response for account listing — contains repositories. +/// API response for listing products under an account. #[derive(Debug, Deserialize)] pub struct AccountResponse { #[serde(default)] - pub repositories: Vec, + pub products: Vec, } #[derive(Debug, Deserialize)] -pub struct AccountRepository { - pub repository_id: String, +pub struct AccountProduct { + pub product_id: String, } // -- Client implementation -- @@ -241,7 +241,7 @@ impl SourceApiClient { &self, account_id: &str, ) -> Result { - let url = format!("{}/api/v1/accounts/{}", self.api_url, account_id); + let url = format!("{}/api/v1/products/{}", self.api_url, account_id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); let cache = (self.account_cache_ttl > 0).then(|| CacheOptions { diff --git a/crates/libs/source-coop/src/resolver.rs b/crates/libs/source-coop/src/resolver.rs index f56fda5..278b9bb 100644 --- a/crates/libs/source-coop/src/resolver.rs +++ b/crates/libs/source-coop/src/resolver.rs @@ -206,9 +206,9 @@ impl SourceCoopResolver { .await?; let prefixes: Vec = account - .repositories + .products .iter() - .map(|r| format!("{}/", r.repository_id)) + .map(|p| format!("{}/", p.product_id)) .collect(); let xml = synthetic_list_objects_v2_xml(account_id, &prefixes); From 99167baba65c9d946292ddebabac693c3826b305 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 21:51:08 -0500 Subject: [PATCH 19/82] chore: add source api logging --- crates/runtimes/cf-workers/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 480b80c..092816c 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -72,6 +72,11 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { )) })?; +tracing::info!( + source_api_url = source_api_url.to_string(), + "SOURCE_API_URL set, using Source Cooperative API resolver" + ); + let mut cache_ttls = CacheTtls::default(); if let Ok(v) = env.var("SOURCE_CACHE_TTL_PRODUCT") { if let Ok(n) = v.to_string().parse::() { From b9280c0fea4f621510da5a611b85dcacc21ec9c7 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 22:06:40 -0500 Subject: [PATCH 20/82] fix: rename connection auth field --- crates/libs/source-coop/src/api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/libs/source-coop/src/api.rs b/crates/libs/source-coop/src/api.rs index 978612f..18d9c0e 100644 --- a/crates/libs/source-coop/src/api.rs +++ b/crates/libs/source-coop/src/api.rs @@ -104,6 +104,7 @@ pub struct ConnectionDetails { #[derive(Debug, Deserialize)] #[allow(dead_code)] pub struct ConnectionAuth { + #[serde(alias = "type")] pub auth_type: String, pub access_key_id: Option, pub secret_access_key: Option, From aa502fe32ce727256c279bf81a9d5893e8af0048 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 22:34:52 -0500 Subject: [PATCH 21/82] chore: cargo fmt --- Cargo.lock | 62 +++++++++---------- crates/libs/source-coop/src/resolver.rs | 33 +++++----- crates/runtimes/cf-workers/Cargo.toml | 2 +- crates/runtimes/cf-workers/src/body.rs | 4 +- crates/runtimes/cf-workers/src/client.rs | 17 ++--- .../cf-workers/src/fetch_connector.rs | 9 +-- crates/runtimes/cf-workers/src/lib.rs | 31 ++++------ .../runtimes/cf-workers/src/tracing_layer.rs | 5 +- crates/runtimes/cf-workers/wrangler.toml | 9 +-- 9 files changed, 78 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c0a19b..14f0581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2078,37 +2078,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "s3-proxy-cf-workers" -version = "0.1.0" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "console_error_panic_hook", - "futures", - "getrandom 0.2.17", - "getrandom 0.3.4", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "js-sys", - "object_store", - "quick-xml 0.37.5", - "s3-proxy-core", - "s3-proxy-source-coop", - "s3-proxy-sts", - "serde", - "serde_json", - "thiserror", - "tracing", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "worker", -] - [[package]] name = "s3-proxy-core" version = "0.1.0" @@ -2156,6 +2125,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "s3-proxy-rs" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "console_error_panic_hook", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "js-sys", + "object_store", + "quick-xml 0.37.5", + "s3-proxy-core", + "s3-proxy-source-coop", + "s3-proxy-sts", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "worker", +] + [[package]] name = "s3-proxy-server" version = "0.1.0" diff --git a/crates/libs/source-coop/src/resolver.rs b/crates/libs/source-coop/src/resolver.rs index 278b9bb..951b9b6 100644 --- a/crates/libs/source-coop/src/resolver.rs +++ b/crates/libs/source-coop/src/resolver.rs @@ -167,10 +167,7 @@ impl SourceCoopResolver { .get_permissions(account_id, repo_id, &sig.access_key_id) .await?; - let is_write = matches!( - *method, - Method::PUT | Method::POST | Method::DELETE - ); + let is_write = matches!(*method, Method::PUT | Method::POST | Method::DELETE); if is_write && !perms.write { tracing::warn!( @@ -196,14 +193,18 @@ impl SourceCoopResolver { } /// Handle `GET /{account_id}` — synthetic account listing. - async fn handle_account_listing( - &self, - account_id: &str, - ) -> Result { - let account = self - .api_client - .list_account_repos(account_id) - .await?; + async fn handle_account_listing(&self, account_id: &str) -> Result { + tracing::info!( + account_id = account_id, + "handling account listing for account" + ); + let account = self.api_client.list_account_repos(account_id).await?; + + tracing::info!( + account_id = account_id, + repo_count = account.products.len(), + "fetched account listing from Source API" + ); let prefixes: Vec = account .products @@ -324,8 +325,7 @@ impl RequestResolver for SourceCoopResolver { let bucket_config = self.resolve_bucket_config(account_id, repo_id).await?; // Build the S3 operation - let operation = - build_s3_operation(method, bucket_name, key.to_string(), query)?; + let operation = build_s3_operation(method, bucket_name, key.to_string(), query)?; // For list operations, apply list rewrite let list_rewrite = if key.is_empty() { @@ -414,10 +414,7 @@ fn rewrite_list_prefix(query: &str, repo_id: &str) -> String { .map(|(k, v)| { if k == "prefix" { let prefix_to_strip = format!("{}/", repo_id); - let new_v = v - .strip_prefix(&prefix_to_strip) - .unwrap_or(&v) - .to_string(); + let new_v = v.strip_prefix(&prefix_to_strip).unwrap_or(&v).to_string(); (k.to_string(), new_v) } else { (k.to_string(), v.to_string()) diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index 3571c5d..b2887c4 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-cf-workers" +name = "s3-proxy-rs" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs index 0a0855c..c8383a8 100644 --- a/crates/runtimes/cf-workers/src/body.rs +++ b/crates/runtimes/cf-workers/src/body.rs @@ -12,9 +12,7 @@ use worker::{Headers, Response}; /// /// Only handles `Bytes` and `Empty` bodies (LIST XML, errors, multipart XML). /// Streaming Forward responses are built directly in `lib.rs`. -pub fn build_worker_response( - result: ProxyResult, -) -> Result { +pub fn build_worker_response(result: ProxyResult) -> Result { let resp_headers = Headers::new(); for (key, value) in result.headers.iter() { if let Ok(v) = value.to_str() { diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 6dae170..b4d1aa6 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -9,7 +9,9 @@ use bytes::Bytes; use http::HeaderMap; use object_store::signer::Signer; use object_store::ObjectStore; -use s3_proxy_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder}; +use s3_proxy_core::backend::{ + build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder, +}; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::BucketConfig; use s3_proxy_source_coop::api::{CacheOptions, HttpClient}; @@ -162,16 +164,15 @@ impl ProxyBackend for WorkerBackend { init.set_body(&uint8.into()); } - let ws_request = - web_sys::Request::new_with_str_and_init(&url, &init).map_err(|e| { - ProxyError::BackendError(format!("failed to create request: {:?}", e)) - })?; + let ws_request = web_sys::Request::new_with_str_and_init(&url, &init) + .map_err(|e| ProxyError::BackendError(format!("failed to create request: {:?}", e)))?; // Fetch via worker let worker_req: worker::Request = ws_request.into(); - let mut worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { - ProxyError::BackendError(format!("fetch failed: {}", e)) - })?; + let mut worker_resp = Fetch::Request(worker_req) + .send() + .await + .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; let status = worker_resp.status_code(); diff --git a/crates/runtimes/cf-workers/src/fetch_connector.rs b/crates/runtimes/cf-workers/src/fetch_connector.rs index 0f5790f..aea686b 100644 --- a/crates/runtimes/cf-workers/src/fetch_connector.rs +++ b/crates/runtimes/cf-workers/src/fetch_connector.rs @@ -37,10 +37,7 @@ impl HttpConnector for FetchConnector { struct FetchService; impl FetchService { - async fn do_fetch( - &self, - worker_req: worker::Request, - ) -> Result { + async fn do_fetch(&self, worker_req: worker::Request) -> Result { let (tx, rx) = oneshot::channel(); spawn_local(async move { @@ -56,9 +53,7 @@ impl FetchService { }) } - async fn fetch_inner( - worker_req: worker::Request, - ) -> Result { + async fn fetch_inner(worker_req: worker::Request) -> Result { let mut resp = worker::Fetch::Request(worker_req) .send() .await diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 092816c..e55f633 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -72,7 +72,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { )) })?; -tracing::info!( + tracing::info!( source_api_url = source_api_url.to_string(), "SOURCE_API_URL set, using Source Cooperative API resolver" ); @@ -154,9 +154,7 @@ async fn handle_action( query: Option<&str>, headers: &http::HeaderMap, ) -> Result { - let action = handler - .resolve_request(method, path, query, headers) - .await; + let action = handler.resolve_request(method, path, query, headers).await; match action { HandlerAction::Response(result) => build_worker_response(result), @@ -174,10 +172,7 @@ async fn handle_action( /// For PUT: passes the original JS `ReadableStream` body directly to fetch. /// For GET: returns the response `ReadableStream` directly to the client. /// Zero Rust stream involvement — bytes never cross the WASM boundary. -async fn execute_forward( - req: &Request, - fwd: ForwardRequest, -) -> Result { +async fn execute_forward(req: &Request, fwd: ForwardRequest) -> Result { // Build request headers let ws_headers = web_sys::Headers::new() .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; @@ -200,15 +195,15 @@ async fn execute_forward( } } - let ws_request = - web_sys::Request::new_with_str_and_init(fwd.url.as_str(), &init) - .map_err(|e| worker::Error::RustError(format!("request error: {:?}", e)))?; + let ws_request = web_sys::Request::new_with_str_and_init(fwd.url.as_str(), &init) + .map_err(|e| worker::Error::RustError(format!("request error: {:?}", e)))?; // Execute fetch let worker_req: worker::Request = ws_request.into(); - let worker_resp = Fetch::Request(worker_req).send().await.map_err(|e| { - worker::Error::RustError(format!("forward fetch failed: {}", e)) - })?; + let worker_resp = Fetch::Request(worker_req) + .send() + .await + .map_err(|e| worker::Error::RustError(format!("forward fetch failed: {}", e)))?; let status = worker_resp.status_code(); let ws_response: web_sys::Response = worker_resp.into(); @@ -229,11 +224,9 @@ async fn execute_forward( resp_init.set_headers(&ws_resp_headers.into()); let body = ws_response.body(); - let response = web_sys::Response::new_with_opt_readable_stream_and_init( - body.as_ref(), - &resp_init, - ) - .map_err(|e| worker::Error::RustError(format!("response error: {:?}", e)))?; + let response = + web_sys::Response::new_with_opt_readable_stream_and_init(body.as_ref(), &resp_init) + .map_err(|e| worker::Error::RustError(format!("response error: {:?}", e)))?; Ok(response.into()) } diff --git a/crates/runtimes/cf-workers/src/tracing_layer.rs b/crates/runtimes/cf-workers/src/tracing_layer.rs index b241177..8aa8332 100644 --- a/crates/runtimes/cf-workers/src/tracing_layer.rs +++ b/crates/runtimes/cf-workers/src/tracing_layer.rs @@ -7,8 +7,8 @@ //! `worker::console_log!` / `console_error!` / `console_warn!`. use tracing::field::{Field, Visit}; -use tracing::{Event, Level, Metadata, Subscriber}; use tracing::span; +use tracing::{Event, Level, Metadata, Subscriber}; /// A minimal tracing subscriber that logs to the Workers console. /// @@ -98,8 +98,7 @@ impl Visit for MessageVisitor { if field.name() == "message" { self.message = value.to_string(); } else { - self.fields - .push(format!("{}=\"{}\"", field.name(), value)); + self.fields.push(format!("{}=\"{}\"", field.name(), value)); } } diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index e327d0d..e48050c 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -1,11 +1,12 @@ -name = "s3-proxy-cf-workers" -main = "build/worker/shim.mjs" compatibility_date = "2024-09-23" +main = "build/worker/shim.mjs" +name = "s3-proxy-cf-workers" [build] command = "cargo install worker-build && worker-build --release" [vars] +SOURCE_API_URL = "https://staging.source.coop" VIRTUAL_HOST_DOMAIN = "s3.local" # For production, consider storing this in Workers KV or a Secrets binding. @@ -41,8 +42,8 @@ secret_access_key = "minioadmin" [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true -backend_type = "s3" backend_prefix = "cholmes/" +backend_type = "s3" name = "cholmes" [vars.PROXY_CONFIG.buckets.backend_options] @@ -54,8 +55,8 @@ skip_signature = "true" [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true -backend_type = "s3" backend_prefix = "harvard-lil/" +backend_type = "s3" name = "harvard-lil" [vars.PROXY_CONFIG.buckets.backend_options] From 76f0ed9b22cfe2efcb95664b2762433fefe44fe4 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 23 Feb 2026 22:34:59 -0500 Subject: [PATCH 22/82] chore: ignore .env --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5666652..95fe15f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store scripts/task_definition.json target -.wrangler \ No newline at end of file +.wrangler +crates/runtimes/cf-workers/.env From 467534b78040c07e0a653df3cf214e150161ce2c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 08:02:43 -0500 Subject: [PATCH 23/82] chore: update crate and worker name --- Cargo.lock | 62 ++++++++++++------------ crates/runtimes/cf-workers/Cargo.toml | 2 +- crates/runtimes/cf-workers/wrangler.toml | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14f0581..8c0a19b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2078,6 +2078,37 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s3-proxy-cf-workers" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "console_error_panic_hook", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "js-sys", + "object_store", + "quick-xml 0.37.5", + "s3-proxy-core", + "s3-proxy-source-coop", + "s3-proxy-sts", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "worker", +] + [[package]] name = "s3-proxy-core" version = "0.1.0" @@ -2125,37 +2156,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "s3-proxy-rs" -version = "0.1.0" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "console_error_panic_hook", - "futures", - "getrandom 0.2.17", - "getrandom 0.3.4", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "js-sys", - "object_store", - "quick-xml 0.37.5", - "s3-proxy-core", - "s3-proxy-source-coop", - "s3-proxy-sts", - "serde", - "serde_json", - "thiserror", - "tracing", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "worker", -] - [[package]] name = "s3-proxy-server" version = "0.1.0" diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index b2887c4..3571c5d 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-rs" +name = "s3-proxy-cf-workers" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index e48050c..737479f 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -1,6 +1,6 @@ compatibility_date = "2024-09-23" main = "build/worker/shim.mjs" -name = "s3-proxy-cf-workers" +name = "s3-proxy-rs" [build] command = "cargo install worker-build && worker-build --release" From c66ae11b2bd251812124b0f5cae029e9b6b58f40 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 08:03:03 -0500 Subject: [PATCH 24/82] chore: cargo clippy --fix --- crates/libs/core/src/s3/request.rs | 12 ++++++------ crates/libs/oidc-provider/src/cache.rs | 6 ++++++ crates/libs/source-coop/src/api.rs | 8 ++++---- crates/runtimes/cf-workers/src/client.rs | 2 +- crates/runtimes/server/src/server.rs | 2 +- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/libs/core/src/s3/request.rs b/crates/libs/core/src/s3/request.rs index 731f563..a18d3e0 100644 --- a/crates/libs/core/src/s3/request.rs +++ b/crates/libs/core/src/s3/request.rs @@ -51,8 +51,8 @@ pub fn build_s3_operation( let has_uploads = query_params.iter().any(|(k, _)| k == "uploads"); - match method { - &Method::GET => { + match *method { + Method::GET => { if key.is_empty() { // ListBucket — pass the raw query string through so the proxy // can forward all list params (prefix, delimiter, max-keys, @@ -65,8 +65,8 @@ pub fn build_s3_operation( Ok(S3Operation::GetObject { bucket, key }) } } - &Method::HEAD => Ok(S3Operation::HeadObject { bucket, key }), - &Method::PUT => { + Method::HEAD => Ok(S3Operation::HeadObject { bucket, key }), + Method::PUT => { if let Some(upload_id) = upload_id { let part_number = query_params .iter() @@ -84,7 +84,7 @@ pub fn build_s3_operation( Ok(S3Operation::PutObject { bucket, key }) } } - &Method::POST => { + Method::POST => { if has_uploads { Ok(S3Operation::CreateMultipartUpload { bucket, key }) } else if let Some(upload_id) = upload_id { @@ -99,7 +99,7 @@ pub fn build_s3_operation( )) } } - &Method::DELETE => { + Method::DELETE => { if let Some(upload_id) = upload_id { Ok(S3Operation::AbortMultipartUpload { bucket, diff --git a/crates/libs/oidc-provider/src/cache.rs b/crates/libs/oidc-provider/src/cache.rs index 3ff2a2f..9e619d7 100644 --- a/crates/libs/oidc-provider/src/cache.rs +++ b/crates/libs/oidc-provider/src/cache.rs @@ -20,6 +20,12 @@ pub struct CredentialCache { entries: Mutex>>, } +impl Default for CredentialCache { + fn default() -> Self { + Self::new() + } +} + impl CredentialCache { pub fn new() -> Self { Self { diff --git a/crates/libs/source-coop/src/api.rs b/crates/libs/source-coop/src/api.rs index 18d9c0e..68c420d 100644 --- a/crates/libs/source-coop/src/api.rs +++ b/crates/libs/source-coop/src/api.rs @@ -178,7 +178,7 @@ impl SourceApiClient { ); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); - let cache = (self.product_cache_ttl > 0).then(|| CacheOptions { + let cache = (self.product_cache_ttl > 0).then_some(CacheOptions { cache_ttl: self.product_cache_ttl, cache_key: None, }); @@ -190,7 +190,7 @@ impl SourceApiClient { let url = format!("{}/api/v1/data-connections/{}", self.api_url, id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); - let cache = (self.data_connection_cache_ttl > 0).then(|| CacheOptions { + let cache = (self.data_connection_cache_ttl > 0).then_some(CacheOptions { cache_ttl: self.data_connection_cache_ttl, cache_key: None, }); @@ -202,7 +202,7 @@ impl SourceApiClient { let url = format!("{}/api/v1/api-keys/{}/auth", self.api_url, access_key_id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); - let cache = (self.api_key_cache_ttl > 0).then(|| CacheOptions { + let cache = (self.api_key_cache_ttl > 0).then_some(CacheOptions { cache_ttl: self.api_key_cache_ttl, cache_key: None, }); @@ -245,7 +245,7 @@ impl SourceApiClient { let url = format!("{}/api/v1/products/{}", self.api_url, account_id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); - let cache = (self.account_cache_ttl > 0).then(|| CacheOptions { + let cache = (self.account_cache_ttl > 0).then_some(CacheOptions { cache_ttl: self.account_cache_ttl, cache_key: None, }); diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index b4d1aa6..557ef2b 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -88,7 +88,7 @@ impl HttpClient for WorkerHttpClient { .await .map_err(|e| ProxyError::Internal(format!("failed to read text: {}", e)))?; - if status < 200 || status >= 300 { + if !(200..300).contains(&status) { return Err(ProxyError::BackendError(format!( "API request to {} returned status {}", url, status diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 9d9ce50..a8d07a8 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -178,7 +178,7 @@ async fn forward_to_backend( let body_stream = backend_resp.bytes_stream(); let framed = body_stream .map_ok(Frame::data) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())); + .map_err(|e| std::io::Error::other(e.to_string())); let body: ServerResponseBody = Either::Left(StreamBody::new( Box::pin(framed) as Pin, std::io::Error>> + Send>> )); From c3c6317649a03cae22a9a127878b744802521d9d Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 08:06:14 -0500 Subject: [PATCH 25/82] chore: cargo fmt --- crates/libs/core/src/backend.rs | 128 +++++++++--------- crates/libs/core/src/config/dynamodb.rs | 13 +- crates/libs/core/src/config/mod.rs | 14 +- crates/libs/core/src/config/postgres.rs | 3 +- crates/libs/core/src/proxy.rs | 107 ++++++++------- crates/libs/core/src/resolver.rs | 15 +- crates/libs/core/src/s3/list_rewrite.rs | 20 +-- crates/libs/core/src/s3/request.rs | 8 +- crates/libs/core/src/types.rs | 3 +- crates/libs/oidc-provider/src/exchange/aws.rs | 24 ++-- .../libs/oidc-provider/src/exchange/azure.rs | 15 +- crates/libs/oidc-provider/src/exchange/gcp.rs | 28 ++-- crates/libs/oidc-provider/src/exchange/mod.rs | 7 +- crates/libs/oidc-provider/src/jwt.rs | 17 ++- crates/libs/oidc-provider/src/lib.rs | 9 +- crates/libs/sts/src/jwks.rs | 24 ++-- crates/libs/sts/src/request.rs | 3 +- crates/runtimes/server/src/body.rs | 7 +- crates/runtimes/server/src/server.rs | 22 +-- 19 files changed, 237 insertions(+), 230 deletions(-) diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index d6272b9..4dfe211 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -86,49 +86,46 @@ impl StoreBuilder { /// Build the final `ObjectStore`. pub fn build(self) -> Result, ProxyError> { match self { - StoreBuilder::S3(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 store: {}", e)))?, - )), + StoreBuilder::S3(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build S3 store: {}", e)) + })?)), #[cfg(feature = "azure")] - StoreBuilder::Azure(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build Azure store: {}", e)))?, - )), + StoreBuilder::Azure(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build Azure store: {}", e)) + })?)), #[cfg(feature = "gcp")] - StoreBuilder::Gcs(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build GCS store: {}", e)))?, - )), + StoreBuilder::Gcs(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build GCS store: {}", e)) + })?)), } } /// Build a `Signer` for presigned URL generation. pub fn build_signer(self) -> Result, ProxyError> { match self { - StoreBuilder::S3(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build S3 signer: {}", e)))?, - )), + StoreBuilder::S3(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build S3 signer: {}", e)) + })?)), #[cfg(feature = "azure")] - StoreBuilder::Azure(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build Azure signer: {}", e)))?, - )), + StoreBuilder::Azure(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build Azure signer: {}", e)) + })?)), #[cfg(feature = "gcp")] - StoreBuilder::Gcs(b) => Ok(Arc::new( - b.build() - .map_err(|e| ProxyError::ConfigError(format!("failed to build GCS signer: {}", e)))?, - )), + StoreBuilder::Gcs(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build GCS signer: {}", e)) + })?)), } } } /// Create a [`StoreBuilder`] from a [`BucketConfig`], dispatching on `backend_type`. fn create_builder(config: &BucketConfig) -> Result { - let backend_type = config - .parsed_backend_type() - .ok_or_else(|| ProxyError::ConfigError(format!("unsupported backend_type: '{}'", config.backend_type)))?; + let backend_type = config.parsed_backend_type().ok_or_else(|| { + ProxyError::ConfigError(format!( + "unsupported backend_type: '{}'", + config.backend_type + )) + })?; match backend_type { BackendType::S3 => { @@ -151,11 +148,9 @@ fn create_builder(config: &BucketConfig) -> Result { Ok(StoreBuilder::Azure(b)) } #[cfg(not(feature = "azure"))] - BackendType::Azure => { - Err(ProxyError::ConfigError( - "Azure backend support not enabled (requires 'azure' feature)".into(), - )) - } + BackendType::Azure => Err(ProxyError::ConfigError( + "Azure backend support not enabled (requires 'azure' feature)".into(), + )), #[cfg(feature = "gcp")] BackendType::Gcs => { let mut b = GoogleCloudStorageBuilder::new(); @@ -167,11 +162,9 @@ fn create_builder(config: &BucketConfig) -> Result { Ok(StoreBuilder::Gcs(b)) } #[cfg(not(feature = "gcp"))] - BackendType::Gcs => { - Err(ProxyError::ConfigError( - "GCS backend support not enabled (requires 'gcp' feature)".into(), - )) - } + BackendType::Gcs => Err(ProxyError::ConfigError( + "GCS backend support not enabled (requires 'gcp' feature)".into(), + )), } } @@ -180,7 +173,10 @@ fn create_builder(config: &BucketConfig) -> Result { /// The `configure` closure lets each runtime inject its HTTP connector: /// - Server runtime passes `|b| b` (default connector) /// - CF Workers passes `|b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), .. }` -pub fn build_object_store(config: &BucketConfig, configure: F) -> Result, ProxyError> +pub fn build_object_store( + config: &BucketConfig, + configure: F, +) -> Result, ProxyError> where F: FnOnce(StoreBuilder) -> StoreBuilder, { @@ -195,11 +191,12 @@ where /// which constructs plain URLs without auth parameters, avoiding the /// `InstanceCredentialProvider` → `Instant::now()` panic on WASM. pub fn build_signer(config: &BucketConfig) -> Result, ProxyError> { - let backend_type = config - .parsed_backend_type() - .ok_or_else(|| { - ProxyError::ConfigError(format!("unsupported backend_type: '{}'", config.backend_type)) - })?; + let backend_type = config.parsed_backend_type().ok_or_else(|| { + ProxyError::ConfigError(format!( + "unsupported backend_type: '{}'", + config.backend_type + )) + })?; // Check for credentials — if absent, return unsigned signer to avoid // InstanceCredentialProvider which uses Instant::now() (panics on WASM). @@ -239,11 +236,7 @@ pub struct S3RequestSigner { } impl S3RequestSigner { - pub fn new( - access_key_id: String, - secret_access_key: String, - region: String, - ) -> Self { + pub fn new(access_key_id: String, secret_access_key: String, region: String) -> Self { Self { access_key_id, secret_access_key, @@ -289,21 +282,13 @@ impl S3RequestSigner { let canonical_uri = url.path(); let canonical_querystring = url.query().unwrap_or(""); - let mut signed_header_names: Vec<&str> = headers - .keys() - .map(|k| k.as_str()) - .collect(); + let mut signed_header_names: Vec<&str> = headers.keys().map(|k| k.as_str()).collect(); signed_header_names.sort(); let canonical_headers: String = signed_header_names .iter() .map(|k| { - let v = headers - .get(*k) - .unwrap() - .to_str() - .unwrap_or("") - .trim(); + let v = headers.get(*k).unwrap().to_str().unwrap_or("").trim(); format!("{}:{}\n", k, v) }) .collect(); @@ -312,11 +297,19 @@ impl S3RequestSigner { let canonical_request = format!( "{}\n{}\n{}\n{}\n{}\n{}", - method, canonical_uri, canonical_querystring, canonical_headers, signed_headers, payload_hash + method, + canonical_uri, + canonical_querystring, + canonical_headers, + signed_headers, + payload_hash ); // String to sign - let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, self.region, self.service); + let credential_scope = format!( + "{}/{}/{}/aws4_request", + date_stamp, self.region, self.service + ); use sha2::Digest; let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); @@ -329,15 +322,14 @@ impl S3RequestSigner { // Signing key type HmacSha256 = Hmac; - let mut mac = HmacSha256::new_from_slice( - format!("AWS4{}", self.secret_access_key).as_bytes(), - ) - .map_err(|e| ProxyError::Internal(e.to_string()))?; + let mut mac = + HmacSha256::new_from_slice(format!("AWS4{}", self.secret_access_key).as_bytes()) + .map_err(|e| ProxyError::Internal(e.to_string()))?; mac.update(date_stamp.as_bytes()); let k_date = mac.finalize().into_bytes(); - let mut mac = HmacSha256::new_from_slice(&k_date) - .map_err(|e| ProxyError::Internal(e.to_string()))?; + let mut mac = + HmacSha256::new_from_slice(&k_date).map_err(|e| ProxyError::Internal(e.to_string()))?; mac.update(self.region.as_bytes()); let k_region = mac.finalize().into_bytes(); @@ -381,7 +373,9 @@ struct UnsignedUrlSigner { impl UnsignedUrlSigner { fn from_config(config: &BucketConfig) -> Result { - let endpoint = config.option("endpoint").unwrap_or("https://s3.amazonaws.com"); + let endpoint = config + .option("endpoint") + .unwrap_or("https://s3.amazonaws.com"); let bucket = config.option("bucket_name").unwrap_or(""); Ok(Self { endpoint: endpoint.trim_end_matches('/').to_string(), diff --git a/crates/libs/core/src/config/dynamodb.rs b/crates/libs/core/src/config/dynamodb.rs index 4230df1..fc5c0a2 100644 --- a/crates/libs/core/src/config/dynamodb.rs +++ b/crates/libs/core/src/config/dynamodb.rs @@ -147,10 +147,7 @@ impl ConfigProvider for DynamoDbProvider { .client() .get_item() .table_name(self.table()) - .key( - "PK", - AttributeValue::S(format!("CRED#{}", access_key_id)), - ) + .key("PK", AttributeValue::S(format!("CRED#{}", access_key_id))) .key("SK", AttributeValue::S("LONG_LIVED".into())) .send() .await @@ -175,8 +172,7 @@ impl ConfigProvider for DynamoDbProvider { &self, cred: &TemporaryCredentials, ) -> Result<(), ProxyError> { - let json = - serde_json::to_string(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; + let json = serde_json::to_string(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; // TTL for DynamoDB auto-expiry let ttl_epoch = cred.expiration.timestamp(); @@ -206,10 +202,7 @@ impl ConfigProvider for DynamoDbProvider { .client() .get_item() .table_name(self.table()) - .key( - "PK", - AttributeValue::S(format!("CRED#{}", access_key_id)), - ) + .key("PK", AttributeValue::S(format!("CRED#{}", access_key_id))) .key("SK", AttributeValue::S("TEMPORARY".into())) .send() .await diff --git a/crates/libs/core/src/config/mod.rs b/crates/libs/core/src/config/mod.rs index d96e03c..35b1ec1 100644 --- a/crates/libs/core/src/config/mod.rs +++ b/crates/libs/core/src/config/mod.rs @@ -53,11 +53,19 @@ use std::future::Future; /// (required by Tokio's task spawning), on WASM it's a no-op (allowing `!Send` /// JS interop types). pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { - fn list_buckets(&self) -> impl Future, ProxyError>> + MaybeSend; + fn list_buckets( + &self, + ) -> impl Future, ProxyError>> + MaybeSend; - fn get_bucket(&self, name: &str) -> impl Future, ProxyError>> + MaybeSend; + fn get_bucket( + &self, + name: &str, + ) -> impl Future, ProxyError>> + MaybeSend; - fn get_role(&self, role_id: &str) -> impl Future, ProxyError>> + MaybeSend; + fn get_role( + &self, + role_id: &str, + ) -> impl Future, ProxyError>> + MaybeSend; /// Look up a long-lived credential by its access key ID. fn get_credential( diff --git a/crates/libs/core/src/config/postgres.rs b/crates/libs/core/src/config/postgres.rs index 6c54977..d81a6f6 100644 --- a/crates/libs/core/src/config/postgres.rs +++ b/crates/libs/core/src/config/postgres.rs @@ -123,8 +123,7 @@ impl ConfigProvider for PostgresProvider { &self, cred: &TemporaryCredentials, ) -> Result<(), ProxyError> { - let json = - serde_json::to_value(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; + let json = serde_json::to_value(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; sqlx::query( "INSERT INTO proxy_credentials (access_key_id, credential_type, config_json, expires_at) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index ea171c3..e2440ac 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -18,9 +18,7 @@ use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLO use crate::error::ProxyError; use crate::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use crate::response_body::ProxyResponseBody; -use crate::s3::response::{ - ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents, -}; +use crate::s3::response::{ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents}; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; @@ -153,11 +151,7 @@ where /// Phase 2: Complete a multipart operation with the request body. /// /// Called by the runtime after materializing the body for a `NeedsBody` action. - pub async fn handle_with_body( - &self, - pending: PendingRequest, - body: Bytes, - ) -> ProxyResult { + pub async fn handle_with_body(&self, pending: PendingRequest, body: Bytes) -> ProxyResult { match self.execute_multipart(&pending, body).await { Ok(result) => { tracing::info!( @@ -229,46 +223,59 @@ where ) -> Result { match operation { S3Operation::GetObject { key, .. } => { - let fwd = self.build_forward( - Method::GET, - bucket_config, - key, - original_headers, - &["range", "if-match", "if-none-match", "if-modified-since", "if-unmodified-since"], - ).await?; + let fwd = self + .build_forward( + Method::GET, + bucket_config, + key, + original_headers, + &[ + "range", + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + ], + ) + .await?; tracing::debug!(url = %fwd.url, "GET via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::HeadObject { key, .. } => { - let fwd = self.build_forward( - Method::HEAD, - bucket_config, - key, - original_headers, - &["if-match", "if-none-match", "if-modified-since", "if-unmodified-since"], - ).await?; + let fwd = self + .build_forward( + Method::HEAD, + bucket_config, + key, + original_headers, + &[ + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + ], + ) + .await?; tracing::debug!(url = %fwd.url, "HEAD via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::PutObject { key, .. } => { - let fwd = self.build_forward( - Method::PUT, - bucket_config, - key, - original_headers, - &["content-type", "content-length", "content-md5"], - ).await?; + let fwd = self + .build_forward( + Method::PUT, + bucket_config, + key, + original_headers, + &["content-type", "content-length", "content-md5"], + ) + .await?; tracing::debug!(url = %fwd.url, "PUT via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::DeleteObject { key, .. } => { - let fwd = self.build_forward( - Method::DELETE, - bucket_config, - key, - original_headers, - &[], - ).await?; + let fwd = self + .build_forward(Method::DELETE, bucket_config, key, original_headers, &[]) + .await?; tracing::debug!(url = %fwd.url, "DELETE via presigned URL"); Ok(HandlerAction::Forward(fwd)) } @@ -413,11 +420,7 @@ where let mut headers = HeaderMap::new(); // Forward relevant headers - for header_name in &[ - "content-type", - "content-length", - "content-md5", - ] { + for header_name in &["content-type", "content-length", "content-md5"] { if let Some(val) = pending.original_headers.get(*header_name) { headers.insert(*header_name, val.clone()); } @@ -429,7 +432,13 @@ where hash_payload(&body) }; - sign_s3_request(&pending.method, &backend_url, &mut headers, &pending.bucket_config, &payload_hash)?; + sign_s3_request( + &pending.method, + &backend_url, + &mut headers, + &pending.bucket_config, + &payload_hash, + )?; let raw_resp = self .backend @@ -497,8 +506,8 @@ fn sign_s3_request( let region = config.option("region").unwrap_or("us-east-1"); let has_credentials = !access_key.is_empty() && !secret_key.is_empty(); - let parsed_url = Url::parse(url) - .map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; + let parsed_url = + Url::parse(url).map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; if has_credentials { let signer = S3RequestSigner::new( @@ -577,7 +586,10 @@ fn build_list_xml( let raw_key = obj.location.to_string(); ListContents { key: rewrite_key(&raw_key, &strip_prefix, list_rewrite), - last_modified: obj.last_modified.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(), + last_modified: obj + .last_modified + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(), etag: obj.e_tag.as_deref().unwrap_or("\"\"").to_string(), size: obj.size, storage_class: "STANDARD", @@ -674,7 +686,10 @@ pub fn build_backend_url( part_number, .. } => { - url.push_str(&format!("?partNumber={}&uploadId={}", part_number, upload_id)); + url.push_str(&format!( + "?partNumber={}&uploadId={}", + part_number, upload_id + )); } S3Operation::CompleteMultipartUpload { upload_id, .. } | S3Operation::AbortMultipartUpload { upload_id, .. } => { diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index 803f21a..ddcfc3c 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -122,17 +122,14 @@ impl RequestResolver for DefaultResolver

{ } // Get bucket name and look up config - let bucket_name = operation.bucket() + let bucket_name = operation + .bucket() .ok_or_else(|| ProxyError::InvalidRequest("no bucket in request".into()))?; - let bucket_config = self - .config - .get_bucket(bucket_name) - .await? - .ok_or_else(|| { - tracing::warn!(bucket = %bucket_name, "bucket not found in config"); - ProxyError::BucketNotFound(bucket_name.to_string()) - })?; + let bucket_config = self.config.get_bucket(bucket_name).await?.ok_or_else(|| { + tracing::warn!(bucket = %bucket_name, "bucket not found in config"); + ProxyError::BucketNotFound(bucket_name.to_string()) + })?; tracing::debug!( bucket = %bucket_name, diff --git a/crates/libs/core/src/s3/list_rewrite.rs b/crates/libs/core/src/s3/list_rewrite.rs index aedbb6a..29cb80e 100644 --- a/crates/libs/core/src/s3/list_rewrite.rs +++ b/crates/libs/core/src/s3/list_rewrite.rs @@ -11,18 +11,18 @@ use crate::resolver::ListRewrite; pub fn rewrite_list_response(xml: &str, rewrite: &ListRewrite) -> String { let mut result = xml.to_string(); result = rewrite_xml_element_values(&result, "Key", &rewrite.strip_prefix, &rewrite.add_prefix); - result = rewrite_xml_element_values(&result, "Prefix", &rewrite.strip_prefix, &rewrite.add_prefix); + result = rewrite_xml_element_values( + &result, + "Prefix", + &rewrite.strip_prefix, + &rewrite.add_prefix, + ); result } /// Replace prefix in XML element values: /// `old_prefix/rest` -> `new_prefix/rest` -fn rewrite_xml_element_values( - xml: &str, - tag: &str, - old_prefix: &str, - new_prefix: &str, -) -> String { +fn rewrite_xml_element_values(xml: &str, tag: &str, old_prefix: &str, new_prefix: &str) -> String { let open = format!("<{}>", tag); let close = format!("", tag); let mut result = String::with_capacity(xml.len()); @@ -79,7 +79,11 @@ mod tests { add_prefix: "repo".to_string(), }; let result = rewrite_list_response(xml, &rewrite); - assert!(result.contains("repo/file.csv"), "got: {}", result); + assert!( + result.contains("repo/file.csv"), + "got: {}", + result + ); } #[test] diff --git a/crates/libs/core/src/s3/request.rs b/crates/libs/core/src/s3/request.rs index a18d3e0..d26dd0a 100644 --- a/crates/libs/core/src/s3/request.rs +++ b/crates/libs/core/src/s3/request.rs @@ -20,12 +20,16 @@ pub fn parse_s3_request( if *method == Method::GET { return Ok(S3Operation::ListBuckets); } - return Err(ProxyError::InvalidRequest("unsupported operation on /".into())); + return Err(ProxyError::InvalidRequest( + "unsupported operation on /".into(), + )); } let (bucket, key) = match host_style { HostStyle::Path => parse_path_style(uri_path)?, - HostStyle::VirtualHosted { bucket } => (bucket, uri_path.trim_start_matches('/').to_string()), + HostStyle::VirtualHosted { bucket } => { + (bucket, uri_path.trim_start_matches('/').to_string()) + } }; build_s3_operation(method, bucket, key, query) diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index ac71c91..7da526b 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -240,8 +240,7 @@ impl S3Operation { | S3Operation::CompleteMultipartUpload { key, .. } | S3Operation::AbortMultipartUpload { key, .. } | S3Operation::DeleteObject { key, .. } => key, - S3Operation::ListBucket { .. } - | S3Operation::ListBuckets => "", + S3Operation::ListBucket { .. } | S3Operation::ListBuckets => "", } } } diff --git a/crates/libs/oidc-provider/src/exchange/aws.rs b/crates/libs/oidc-provider/src/exchange/aws.rs index 9c9f88f..7e73216 100644 --- a/crates/libs/oidc-provider/src/exchange/aws.rs +++ b/crates/libs/oidc-provider/src/exchange/aws.rs @@ -47,11 +47,7 @@ impl AwsExchange { } impl CredentialExchange for AwsExchange { - async fn exchange( - &self, - http: &H, - jwt: &str, - ) -> Result { + async fn exchange(&self, http: &H, jwt: &str) -> Result { let form = [ ("Action", "AssumeRoleWithWebIdentity"), ("Version", "2011-06-15"), @@ -60,9 +56,7 @@ impl CredentialExchange for AwsExchange { ("WebIdentityToken", jwt), ]; - let body = http - .post_form(&self.sts_endpoint, &form) - .await?; + let body = http.post_form(&self.sts_endpoint, &form).await?; parse_assume_role_response(&body) } @@ -103,14 +97,12 @@ fn parse_assume_role_response(xml: &str) -> Result Result { let open = format!("<{tag}>"); let close = format!(""); - let start = xml - .find(&open) - .ok_or_else(|| OidcProviderError::ExchangeError(format!("missing <{tag}> in STS response")))? - + open.len(); - let end = xml[start..] - .find(&close) - .ok_or_else(|| OidcProviderError::ExchangeError(format!("missing in STS response")))? - + start; + let start = xml.find(&open).ok_or_else(|| { + OidcProviderError::ExchangeError(format!("missing <{tag}> in STS response")) + })? + open.len(); + let end = xml[start..].find(&close).ok_or_else(|| { + OidcProviderError::ExchangeError(format!("missing in STS response")) + })? + start; Ok(xml[start..end].to_string()) } diff --git a/crates/libs/oidc-provider/src/exchange/azure.rs b/crates/libs/oidc-provider/src/exchange/azure.rs index 855c4be..a45d2af 100644 --- a/crates/libs/oidc-provider/src/exchange/azure.rs +++ b/crates/libs/oidc-provider/src/exchange/azure.rs @@ -43,11 +43,7 @@ impl AzureExchange { } impl CredentialExchange for AzureExchange { - async fn exchange( - &self, - http: &H, - jwt: &str, - ) -> Result { + async fn exchange(&self, http: &H, jwt: &str) -> Result { let form = [ ("grant_type", "client_credentials"), ( @@ -59,9 +55,7 @@ impl CredentialExchange for AzureExchange { ("scope", &self.scope), ]; - let body = http - .post_form(&self.token_endpoint(), &form) - .await?; + let body = http.post_form(&self.token_endpoint(), &form).await?; parse_azure_token_response(&body) } @@ -69,8 +63,9 @@ impl CredentialExchange for AzureExchange { /// Parse an Azure AD token response. fn parse_azure_token_response(json: &str) -> Result { - let parsed: serde_json::Value = serde_json::from_str(json) - .map_err(|e| OidcProviderError::ExchangeError(format!("invalid Azure token response: {e}")))?; + let parsed: serde_json::Value = serde_json::from_str(json).map_err(|e| { + OidcProviderError::ExchangeError(format!("invalid Azure token response: {e}")) + })?; if let Some(err) = parsed.get("error") { let desc = parsed diff --git a/crates/libs/oidc-provider/src/exchange/gcp.rs b/crates/libs/oidc-provider/src/exchange/gcp.rs index e590d10..90a299c 100644 --- a/crates/libs/oidc-provider/src/exchange/gcp.rs +++ b/crates/libs/oidc-provider/src/exchange/gcp.rs @@ -46,14 +46,13 @@ impl GcpExchange { } impl CredentialExchange for GcpExchange { - async fn exchange( - &self, - http: &H, - jwt: &str, - ) -> Result { + async fn exchange(&self, http: &H, jwt: &str) -> Result { // Step 1: Exchange JWT for federated access token via GCP STS let sts_form = [ - ("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"), + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:token-exchange", + ), ("audience", &self.provider_resource_name), ("scope", "https://www.googleapis.com/auth/cloud-platform"), ( @@ -64,9 +63,7 @@ impl CredentialExchange for GcpExchange { ("subject_token", jwt), ]; - let sts_body = http - .post_form(&self.sts_endpoint, &sts_form) - .await?; + let sts_body = http.post_form(&self.sts_endpoint, &sts_form).await?; let federated_token = parse_sts_token_response(&sts_body)?; @@ -114,17 +111,16 @@ fn parse_sts_token_response(json: &str) -> Result { parsed["access_token"] .as_str() .map(|s| s.to_string()) - .ok_or_else(|| OidcProviderError::ExchangeError("missing access_token in STS response".into())) + .ok_or_else(|| { + OidcProviderError::ExchangeError("missing access_token in STS response".into()) + }) } /// Parse the IAM `generateAccessToken` response. fn parse_generate_access_token_response(json: &str) -> Result { - let parsed: serde_json::Value = serde_json::from_str(json) - .map_err(|e| { - OidcProviderError::ExchangeError(format!( - "invalid generateAccessToken response: {e}" - )) - })?; + let parsed: serde_json::Value = serde_json::from_str(json).map_err(|e| { + OidcProviderError::ExchangeError(format!("invalid generateAccessToken response: {e}")) + })?; let access_token = parsed["accessToken"] .as_str() diff --git a/crates/libs/oidc-provider/src/exchange/mod.rs b/crates/libs/oidc-provider/src/exchange/mod.rs index 17b7a85..1d280c9 100644 --- a/crates/libs/oidc-provider/src/exchange/mod.rs +++ b/crates/libs/oidc-provider/src/exchange/mod.rs @@ -14,10 +14,13 @@ use crate::{CloudCredentials, HttpExchange, OidcProviderError}; /// - AWS: `AssumeRoleWithWebIdentity` via STS /// - Azure: Federated token exchange via Azure AD /// - GCP: STS token exchange + `generateAccessToken` via IAM -pub trait CredentialExchange: s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync { +pub trait CredentialExchange: + s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync +{ fn exchange( &self, http: &H, jwt: &str, - ) -> impl std::future::Future> + s3_proxy_core::maybe_send::MaybeSend; + ) -> impl std::future::Future> + + s3_proxy_core::maybe_send::MaybeSend; } diff --git a/crates/libs/oidc-provider/src/jwt.rs b/crates/libs/oidc-provider/src/jwt.rs index 9f2e0ed..2f435e5 100644 --- a/crates/libs/oidc-provider/src/jwt.rs +++ b/crates/libs/oidc-provider/src/jwt.rs @@ -22,8 +22,9 @@ pub struct JwtSigner { impl JwtSigner { /// Create a signer from a PEM-encoded PKCS#8 private key. pub fn from_pem(pem: &str, kid: String, ttl_seconds: i64) -> Result { - let private_key = RsaPrivateKey::from_pkcs8_pem(pem) - .map_err(|e| OidcProviderError::KeyError(format!("failed to parse private key: {e}")))?; + let private_key = RsaPrivateKey::from_pkcs8_pem(pem).map_err(|e| { + OidcProviderError::KeyError(format!("failed to parse private key: {e}")) + })?; Ok(Self { private_key, kid, @@ -75,7 +76,10 @@ impl JwtSigner { }); if let serde_json::Value::Object(ref mut map) = payload { for (k, v) in extra_claims { - map.insert((*k).to_string(), serde_json::Value::String((*v).to_string())); + map.insert( + (*k).to_string(), + serde_json::Value::String((*v).to_string()), + ); } } let payload_b64 = b64.encode(payload.to_string().as_bytes()); @@ -109,7 +113,12 @@ mod tests { let pem = test_key_pem(); let signer = JwtSigner::from_pem(&pem, "test-kid".into(), 300).unwrap(); let token = signer - .sign("my-subject", "https://proxy.example.com", "sts.amazonaws.com", &[]) + .sign( + "my-subject", + "https://proxy.example.com", + "sts.amazonaws.com", + &[], + ) .unwrap(); let parts: Vec<&str> = token.split('.').collect(); diff --git a/crates/libs/oidc-provider/src/lib.rs b/crates/libs/oidc-provider/src/lib.rs index 44e5e9c..d2e9b8e 100644 --- a/crates/libs/oidc-provider/src/lib.rs +++ b/crates/libs/oidc-provider/src/lib.rs @@ -15,8 +15,8 @@ pub mod cache; pub mod discovery; pub mod exchange; -pub mod jwt; pub mod jwks; +pub mod jwt; use std::sync::Arc; @@ -37,12 +37,15 @@ pub struct CloudCredentials { /// /// Each runtime provides its own implementation — `reqwest` on native, /// `Fetch` on Cloudflare Workers. -pub trait HttpExchange: Clone + s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync + 'static { +pub trait HttpExchange: + Clone + s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync + 'static +{ fn post_form( &self, url: &str, form: &[(&str, &str)], - ) -> impl std::future::Future> + s3_proxy_core::maybe_send::MaybeSend; + ) -> impl std::future::Future> + + s3_proxy_core::maybe_send::MaybeSend; } /// Top-level provider that combines signing, exchange, and caching. diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index ede1a68..12cf7a5 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -33,11 +33,10 @@ pub async fn fetch_jwks(issuer: &str) -> Result { let config_url = format!("{}/.well-known/openid-configuration", issuer); let client = reqwest::Client::new(); - let config_resp = client - .get(&config_url) - .send() - .await - .map_err(|e| ProxyError::InvalidOidcToken(format!("failed to fetch OIDC config: {}", e)))?; + let config_resp = + client.get(&config_url).send().await.map_err(|e| { + ProxyError::InvalidOidcToken(format!("failed to fetch OIDC config: {}", e)) + })?; let config: serde_json::Value = config_resp .json() @@ -114,10 +113,7 @@ pub fn verify_token( let header_bytes = base64url_decode(header_b64)?; let header: serde_json::Value = serde_json::from_slice(&header_bytes) .map_err(|e| ProxyError::InvalidOidcToken(format!("invalid JWT header JSON: {}", e)))?; - let alg = header - .get("alg") - .and_then(|v| v.as_str()) - .unwrap_or(""); + let alg = header.get("alg").and_then(|v| v.as_str()).unwrap_or(""); if alg != "RS256" { return Err(ProxyError::InvalidOidcToken(format!( "unsupported JWT algorithm: {}", @@ -134,7 +130,9 @@ pub fn verify_token( let signed_content = format!("{}.{}", header_b64, payload_b64); verifying_key .verify(signed_content.as_bytes(), &signature) - .map_err(|e| ProxyError::InvalidOidcToken(format!("JWT signature verification failed: {}", e)))?; + .map_err(|e| { + ProxyError::InvalidOidcToken(format!("JWT signature verification failed: {}", e)) + })?; // Decode and validate claims let payload_bytes = base64url_decode(payload_b64)?; @@ -154,9 +152,9 @@ pub fn verify_token( if let Some(ref required_aud) = role.required_audience { let aud_valid = match claims.get("aud") { Some(serde_json::Value::String(aud)) => aud == required_aud, - Some(serde_json::Value::Array(auds)) => { - auds.iter().any(|a| a.as_str() == Some(required_aud.as_str())) - } + Some(serde_json::Value::Array(auds)) => auds + .iter() + .any(|a| a.as_str() == Some(required_aud.as_str())), _ => false, }; if !aud_valid { diff --git a/crates/libs/sts/src/request.rs b/crates/libs/sts/src/request.rs index d93e864..3714bbb 100644 --- a/crates/libs/sts/src/request.rs +++ b/crates/libs/sts/src/request.rs @@ -79,7 +79,8 @@ mod tests { #[test] fn test_sts_request_with_duration() { - let query = "Action=AssumeRoleWithWebIdentity&RoleArn=r&WebIdentityToken=t&DurationSeconds=7200"; + let query = + "Action=AssumeRoleWithWebIdentity&RoleArn=r&WebIdentityToken=t&DurationSeconds=7200"; let result = try_parse_sts_request(Some(query)).unwrap().unwrap(); assert_eq!(result.duration_seconds, Some(7200)); } diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs index 671384a..b369f55 100644 --- a/crates/runtimes/server/src/body.rs +++ b/crates/runtimes/server/src/body.rs @@ -13,11 +13,8 @@ use s3_proxy_core::response_body::ProxyResponseBody; use std::pin::Pin; /// A boxed streaming body type that erases concrete stream types. -type BoxedStreamBody = StreamBody< - Pin< - Box, std::io::Error>> + Send>, - >, ->; +type BoxedStreamBody = + StreamBody, std::io::Error>> + Send>>>; /// The server response body type: either a stream (Forward) or fixed bytes/empty (Response). pub type ServerResponseBody = Either, Empty>>; diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index a8d07a8..cbb3787 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -10,7 +10,9 @@ use hyper::body::{Frame, Incoming}; use hyper::service::service_fn; use hyper_util::rt::{TokioExecutor, TokioIo}; use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; +use s3_proxy_core::proxy::{ + ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, +}; use s3_proxy_core::resolver::DefaultResolver; use std::net::SocketAddr; use std::pin::Pin; @@ -52,7 +54,10 @@ impl Default for ServerConfig { /// run(config, server_config).await.unwrap(); /// } /// ``` -pub async fn run

(config: P, server_config: ServerConfig) -> Result<(), Box> +pub async fn run

( + config: P, + server_config: ServerConfig, +) -> Result<(), Box> where P: ConfigProvider + Send + Sync + 'static, { @@ -127,9 +132,7 @@ where match action { HandlerAction::Response(result) => build_hyper_response(result), - HandlerAction::Forward(fwd) => { - forward_to_backend(client, fwd, incoming_body).await - } + HandlerAction::Forward(fwd) => forward_to_backend(client, fwd, incoming_body).await, HandlerAction::NeedsBody(pending) => { let body = incoming_body.collect().await?.to_bytes(); let result = handler.handle_with_body(pending, body).await; @@ -153,9 +156,7 @@ async fn forward_to_backend( // Attach streaming body for PUT if fwd.method == http::Method::PUT { let body_stream = BodyStream::new(incoming_body) - .try_filter_map(|frame| async move { - Ok(frame.into_data().ok()) - }); + .try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); req_builder = req_builder.body(reqwest::Body::wrap_stream(body_stream)); } @@ -179,9 +180,8 @@ async fn forward_to_backend( let framed = body_stream .map_ok(Frame::data) .map_err(|e| std::io::Error::other(e.to_string())); - let body: ServerResponseBody = Either::Left(StreamBody::new( - Box::pin(framed) as Pin, std::io::Error>> + Send>> - )); + let body: ServerResponseBody = Either::Left(StreamBody::new(Box::pin(framed) + as Pin, std::io::Error>> + Send>>)); let mut builder = Response::builder().status(status); for (k, v) in resp_headers.iter() { From 71594ad7a80873b55541858b1e48382c62954b03 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 08:08:07 -0500 Subject: [PATCH 26/82] chore: add helper utils --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index f15b63d..eb1cc63 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,16 @@ check: cargo check cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +fmt: + cargo fmt -- --check +fmt\:fix: + cargo fmt + +clippy: + cargo clippy -- -D warnings +clippy\:fix: + cargo clippy --fix --allow-dirty --allow-staged + test: cargo test @@ -12,3 +22,4 @@ run\:server: run\:workers: npx wrangler dev --cwd crates/runtimes/cf-workers + From 9c3718905f51059617160906b5ccd76308795874 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 09:27:09 -0500 Subject: [PATCH 27/82] fix: validate sigv4 signatures --- crates/libs/core/src/auth.rs | 531 ++++++++++++++++++++++++++++++- crates/libs/core/src/resolver.rs | 4 +- 2 files changed, 531 insertions(+), 4 deletions(-) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 0292576..e02afd0 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -158,8 +158,12 @@ fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { /// Resolve the identity of an incoming request. /// -/// Checks the Authorization header and resolves it against the config provider. +/// Parses the SigV4 Authorization header, looks up the credential, verifies +/// the signature, and returns the resolved identity. pub async fn resolve_identity( + method: &http::Method, + uri_path: &str, + query_string: &str, headers: &HeaderMap, config: &C, ) -> Result { @@ -170,17 +174,43 @@ pub async fn resolve_identity( let sig = parse_sigv4_auth(auth_header)?; + // The payload hash is sent by the client in x-amz-content-sha256. + // For streaming uploads this is the UNSIGNED-PAYLOAD or + // STREAMING-AWS4-HMAC-SHA256-PAYLOAD sentinel — both are valid + // canonical-request inputs per the SigV4 spec. + let payload_hash = headers + .get("x-amz-content-sha256") + .and_then(|v| v.to_str().ok()) + .unwrap_or("UNSIGNED-PAYLOAD"); + // Check for temporary credentials first (session token present) if headers.get("x-amz-security-token").is_some() { if let Some(temp_cred) = config.get_temporary_credential(&sig.access_key_id).await? { - // Verify session token matches + // Verify session token matches (constant-time to avoid timing leaks) let session_token = headers .get("x-amz-security-token") .and_then(|v| v.to_str().ok()) .unwrap_or(""); - if session_token != temp_cred.session_token { + if !constant_time_eq( + session_token.as_bytes(), + temp_cred.session_token.as_bytes(), + ) { return Err(ProxyError::AccessDenied); } + + // Verify SigV4 signature + if !verify_sigv4_signature( + method, + uri_path, + query_string, + headers, + &sig, + &temp_cred.secret_access_key, + payload_hash, + )? { + return Err(ProxyError::SignatureDoesNotMatch); + } + return Ok(ResolvedIdentity::Temporary { credentials: temp_cred, }); @@ -198,12 +228,507 @@ pub async fn resolve_identity( return Err(ProxyError::ExpiredCredentials); } } + + // Verify SigV4 signature + if !verify_sigv4_signature( + method, + uri_path, + query_string, + headers, + &sig, + &cred.secret_access_key, + payload_hash, + )? { + return Err(ProxyError::SignatureDoesNotMatch); + } + return Ok(ResolvedIdentity::LongLived { credential: cred }); } Err(ProxyError::AccessDenied) } +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ + AccessScope, Action, BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials, + }; + + // ── Mock config provider ────────────────────────────────────────── + + #[derive(Clone)] + struct MockConfig { + credentials: Vec, + temp_credentials: Vec, + } + + impl MockConfig { + fn with_credential(secret: &str) -> Self { + Self { + credentials: vec![StoredCredential { + access_key_id: "AKIAIOSFODNN7EXAMPLE".into(), + secret_access_key: secret.into(), + principal_name: "test-user".into(), + allowed_scopes: vec![AccessScope { + bucket: "test-bucket".into(), + prefixes: vec![], + actions: vec![Action::GetObject], + }], + created_at: chrono::Utc::now(), + expires_at: None, + enabled: true, + }], + temp_credentials: vec![], + } + } + + fn with_temp_credential(secret: &str, session_token: &str) -> Self { + Self { + credentials: vec![], + temp_credentials: vec![TemporaryCredentials { + access_key_id: "ASIATEMP1234EXAMPLE".into(), + secret_access_key: secret.into(), + session_token: session_token.into(), + expiration: chrono::Utc::now() + chrono::Duration::hours(1), + allowed_scopes: vec![AccessScope { + bucket: "test-bucket".into(), + prefixes: vec![], + actions: vec![Action::GetObject], + }], + assumed_role_id: "role-1".into(), + source_identity: "test".into(), + }], + } + } + + fn empty() -> Self { + Self { + credentials: vec![], + temp_credentials: vec![], + } + } + } + + impl crate::config::ConfigProvider for MockConfig { + async fn list_buckets(&self) -> Result, ProxyError> { + Ok(vec![]) + } + async fn get_bucket(&self, _: &str) -> Result, ProxyError> { + Ok(None) + } + async fn get_role(&self, _: &str) -> Result, ProxyError> { + Ok(None) + } + async fn get_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + Ok(self + .credentials + .iter() + .find(|c| c.access_key_id == access_key_id) + .cloned()) + } + async fn store_temporary_credential(&self, _: &TemporaryCredentials) -> Result<(), ProxyError> { + Ok(()) + } + async fn get_temporary_credential( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { + Ok(self + .temp_credentials + .iter() + .find(|c| c.access_key_id == access_key_id) + .cloned()) + } + } + + // ── Test signing helper ─────────────────────────────────────────── + + /// Build a valid SigV4 Authorization header value for testing. + fn sign_request( + method: &http::Method, + uri_path: &str, + query_string: &str, + headers: &HeaderMap, + access_key_id: &str, + secret_access_key: &str, + date_stamp: &str, + amz_date: &str, + region: &str, + signed_header_names: &[&str], + payload_hash: &str, + ) -> String { + let canonical_headers: String = signed_header_names + .iter() + .map(|name| { + let value = headers + .get(*name) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .trim(); + format!("{}:{}\n", name, value) + }) + .collect(); + + let signed_headers_str = signed_header_names.join(";"); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, uri_path, query_string, canonical_headers, signed_headers_str, payload_hash + ); + + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + let credential_scope = format!("{}/{}/s3/aws4_request", date_stamp, region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, canonical_request_hash + ); + + let k_date = + hmac_sha256(format!("AWS4{}", secret_access_key).as_bytes(), date_stamp.as_bytes()) + .unwrap(); + let k_region = hmac_sha256(&k_date, region.as_bytes()).unwrap(); + let k_service = hmac_sha256(&k_region, b"s3").unwrap(); + let signing_key = hmac_sha256(&k_service, b"aws4_request").unwrap(); + let signature = hex::encode(hmac_sha256(&signing_key, string_to_sign.as_bytes()).unwrap()); + + format!( + "AWS4-HMAC-SHA256 Credential={}/{}/{}/s3/aws4_request, SignedHeaders={}, Signature={}", + access_key_id, date_stamp, region, signed_headers_str, signature + ) + } + + /// Build headers and auth for a simple GET request. + fn make_signed_headers( + access_key_id: &str, + secret_access_key: &str, + ) -> HeaderMap { + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let region = "us-east-1"; + let payload_hash = "UNSIGNED-PAYLOAD"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + + let auth = sign_request( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + access_key_id, + secret_access_key, + date_stamp, + amz_date, + region, + &["host", "x-amz-content-sha256", "x-amz-date"], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + headers + } + + // ── Tests ───────────────────────────────────────────────────────── + + fn run(f: F) -> F::Output { + futures::executor::block_on(f) + } + + #[test] + fn no_auth_header_returns_anonymous() { + run(async { + let headers = HeaderMap::new(); + let config = MockConfig::empty(); + + let identity = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap(); + + assert!(matches!(identity, ResolvedIdentity::Anonymous)); + }); + } + + #[test] + fn valid_signature_resolves_identity() { + run(async { + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let config = MockConfig::with_credential(secret); + let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); + + let identity = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap(); + + assert!(matches!(identity, ResolvedIdentity::LongLived { .. })); + }); + } + + #[test] + fn wrong_signature_is_rejected() { + run(async { + let real_secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; + let config = MockConfig::with_credential(real_secret); + // Sign with wrong secret — access_key_id is correct, signature won't match + let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", wrong_secret); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!( + matches!(err, ProxyError::SignatureDoesNotMatch), + "expected SignatureDoesNotMatch, got: {:?}", + err + ); + }); + } + + #[test] + fn garbage_signature_is_rejected() { + run(async { + let real_secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let config = MockConfig::with_credential(real_secret); + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", "20240101T000000Z".parse().unwrap()); + headers.insert("x-amz-content-sha256", "UNSIGNED-PAYLOAD".parse().unwrap()); + headers.insert( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20240101/us-east-1/s3/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date, \ + Signature=0000000000000000000000000000000000000000000000000000000000000000" + .parse() + .unwrap(), + ); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!(matches!(err, ProxyError::SignatureDoesNotMatch)); + }); + } + + #[test] + fn unknown_access_key_is_rejected() { + run(async { + let config = MockConfig::empty(); + let headers = make_signed_headers("AKIAUNKNOWN000000000", "some-secret"); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!(matches!(err, ProxyError::AccessDenied)); + }); + } + + #[test] + fn temp_credential_valid_signature_and_token() { + run(async { + let secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; + let config = MockConfig::with_temp_credential(secret, session_token); + + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let payload_hash = "UNSIGNED-PAYLOAD"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + headers.insert("x-amz-security-token", session_token.parse().unwrap()); + + let auth = sign_request( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + "ASIATEMP1234EXAMPLE", + secret, + date_stamp, + amz_date, + "us-east-1", + &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + + let identity = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap(); + + assert!(matches!(identity, ResolvedIdentity::Temporary { .. })); + }); + } + + #[test] + fn temp_credential_wrong_session_token_is_rejected() { + run(async { + let secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let real_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; + let wrong_token = "WRONG_TOKEN_VALUE_HERE"; + let config = MockConfig::with_temp_credential(secret, real_token); + + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let payload_hash = "UNSIGNED-PAYLOAD"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + headers.insert("x-amz-security-token", wrong_token.parse().unwrap()); + + let auth = sign_request( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + "ASIATEMP1234EXAMPLE", + secret, + date_stamp, + amz_date, + "us-east-1", + &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!(matches!(err, ProxyError::AccessDenied)); + }); + } + + #[test] + fn temp_credential_wrong_signature_is_rejected() { + run(async { + let real_secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; + let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; + let config = MockConfig::with_temp_credential(real_secret, session_token); + + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let payload_hash = "UNSIGNED-PAYLOAD"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + headers.insert("x-amz-security-token", session_token.parse().unwrap()); + + // Sign with wrong secret — session token is correct but sig won't match + let auth = sign_request( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + "ASIATEMP1234EXAMPLE", + wrong_secret, + date_stamp, + amz_date, + "us-east-1", + &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!( + matches!(err, ProxyError::SignatureDoesNotMatch), + "expected SignatureDoesNotMatch, got: {:?}", + err + ); + }); + } + + #[test] + fn disabled_credential_is_rejected_before_sig_check() { + run(async { + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let mut config = MockConfig::with_credential(secret); + config.credentials[0].enabled = false; + + let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + ) + .await + .unwrap_err(); + + assert!(matches!(err, ProxyError::AccessDenied)); + }); + } +} + /// Check if a resolved identity is authorized to perform an operation. pub fn authorize( identity: &ResolvedIdentity, diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index ddcfc3c..cbe1adc 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -138,7 +138,9 @@ impl RequestResolver for DefaultResolver

{ ); // Authenticate - let identity = auth::resolve_identity(headers, &self.config).await?; + let identity = + auth::resolve_identity(method, path, query.unwrap_or(""), headers, &self.config) + .await?; tracing::debug!(identity = ?identity, "resolved identity"); // Authorize From c40509acbba0bd1145a7024bf49348f809eb14ab Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 09:34:28 -0500 Subject: [PATCH 28/82] Fix: upload_id query string injection Replaced raw format! string interpolation of upload_id and part_number into query strings with url::form_urlencoded::Serializer::append_pair(), which properly percent-encodes special characters (&, =, etc.) in both UploadPart and CompleteMultipartUpload/AbortMultipartUpload URL construction. --- crates/libs/core/src/proxy.rs | 98 +++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index e2440ac..feb2444 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -686,17 +686,105 @@ pub fn build_backend_url( part_number, .. } => { - url.push_str(&format!( - "?partNumber={}&uploadId={}", - part_number, upload_id - )); + let qs = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("partNumber", &part_number.to_string()) + .append_pair("uploadId", upload_id) + .finish(); + url.push('?'); + url.push_str(&qs); } S3Operation::CompleteMultipartUpload { upload_id, .. } | S3Operation::AbortMultipartUpload { upload_id, .. } => { - url.push_str(&format!("?uploadId={}", upload_id)); + let qs = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("uploadId", upload_id) + .finish(); + url.push('?'); + url.push_str(&qs); } _ => {} } Ok(url) } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn test_bucket_config() -> BucketConfig { + let mut backend_options = HashMap::new(); + backend_options.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + backend_options.insert("bucket_name".into(), "my-backend-bucket".into()); + BucketConfig { + name: "test".into(), + backend_type: "s3".into(), + backend_prefix: None, + anonymous_access: false, + allowed_roles: vec![], + backend_options, + } + } + + #[test] + fn upload_id_with_special_chars_is_encoded() { + let config = test_bucket_config(); + let malicious_upload_id = "abc&x-amz-acl=public-read&foo=bar"; + let op = S3Operation::UploadPart { + bucket: "test".into(), + key: "file.bin".into(), + upload_id: malicious_upload_id.into(), + part_number: 1, + }; + + let url = build_backend_url(&config, &op).unwrap(); + + // The & and = characters in upload_id must be percent-encoded so they + // cannot act as query parameter separators/assignments. + let query = url.split_once('?').unwrap().1; + let params: Vec<(String, String)> = + url::form_urlencoded::parse(query.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + // Should be exactly 2 params: partNumber and uploadId + assert_eq!(params.len(), 2, "expected 2 query params, got: {:?}", params); + assert!(params.iter().any(|(k, v)| k == "partNumber" && v == "1")); + assert!(params.iter().any(|(k, v)| k == "uploadId" && v == malicious_upload_id)); + } + + #[test] + fn upload_id_encoded_in_complete_multipart() { + let config = test_bucket_config(); + let op = S3Operation::CompleteMultipartUpload { + bucket: "test".into(), + key: "file.bin".into(), + upload_id: "id&injected=true".into(), + }; + + let url = build_backend_url(&config, &op).unwrap(); + + assert!( + !url.contains("injected=true"), + "upload_id was not encoded: {}", + url + ); + } + + #[test] + fn normal_upload_id_works() { + let config = test_bucket_config(); + let op = S3Operation::UploadPart { + bucket: "test".into(), + key: "file.bin".into(), + upload_id: "2~abcdef1234567890".into(), + part_number: 3, + }; + + let url = build_backend_url(&config, &op).unwrap(); + + assert!(url.starts_with("https://s3.us-east-1.amazonaws.com/my-backend-bucket/file.bin?")); + assert!(url.contains("partNumber=3")); + assert!(url.contains("uploadId=2~abcdef1234567890") || url.contains("uploadId=2%7Eabcdef1234567890")); + } +} From 52ccabcaa0767a03c1af216fd2ea426af77242ff Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 09:38:27 -0500 Subject: [PATCH 29/82] fix: HTTP config provider path traversal Added validate_path_segment() that rejects values containing /, \, \0, .., ., or empty strings. Called before every format!("/path/{}", user_input) interpolation in get_bucket, get_role, get_credential, and get_temporary_credential. --- crates/libs/core/src/config/http.rs | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/libs/core/src/config/http.rs b/crates/libs/core/src/config/http.rs index 01e9436..56ffefe 100644 --- a/crates/libs/core/src/config/http.rs +++ b/crates/libs/core/src/config/http.rs @@ -29,6 +29,26 @@ use crate::error::ProxyError; use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; use std::sync::Arc; +/// Validate that a value is safe to use as a single URL path segment. +/// +/// Rejects values containing `/`, `\`, `..`, null bytes, or that are empty, +/// to prevent path traversal against the config API. +fn validate_path_segment(value: &str, param_name: &str) -> Result<(), ProxyError> { + if value.is_empty() + || value.contains('/') + || value.contains('\\') + || value.contains('\0') + || value == ".." + || value == "." + { + return Err(ProxyError::InvalidRequest(format!( + "invalid {}: contains illegal characters", + param_name + ))); + } + Ok(()) +} + /// Configuration provider that reads from a REST API. #[derive(Clone)] pub struct HttpProvider { @@ -82,6 +102,7 @@ impl ConfigProvider for HttpProvider { } async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + validate_path_segment(name, "bucket name")?; let resp = self .request(&format!("/buckets/{}", name)) .send() @@ -99,6 +120,7 @@ impl ConfigProvider for HttpProvider { } async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + validate_path_segment(role_id, "role ID")?; let resp = self .request(&format!("/roles/{}", role_id)) .send() @@ -119,6 +141,7 @@ impl ConfigProvider for HttpProvider { &self, access_key_id: &str, ) -> Result, ProxyError> { + validate_path_segment(access_key_id, "access key ID")?; let resp = self .request(&format!("/credentials/{}", access_key_id)) .send() @@ -160,6 +183,7 @@ impl ConfigProvider for HttpProvider { &self, access_key_id: &str, ) -> Result, ProxyError> { + validate_path_segment(access_key_id, "access key ID")?; let resp = self .request(&format!("/temporary-credentials/{}", access_key_id)) .send() @@ -176,3 +200,27 @@ impl ConfigProvider for HttpProvider { .map_err(|e| ProxyError::ConfigError(e.to_string())) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_path_segment_rejects_traversal() { + assert!(validate_path_segment("../admin", "test").is_err()); + assert!(validate_path_segment("foo/bar", "test").is_err()); + assert!(validate_path_segment("foo\\bar", "test").is_err()); + assert!(validate_path_segment("..", "test").is_err()); + assert!(validate_path_segment(".", "test").is_err()); + assert!(validate_path_segment("", "test").is_err()); + assert!(validate_path_segment("foo\0bar", "test").is_err()); + } + + #[test] + fn validate_path_segment_accepts_normal_values() { + assert!(validate_path_segment("my-bucket", "test").is_ok()); + assert!(validate_path_segment("AKIAIOSFODNN7EXAMPLE", "test").is_ok()); + assert!(validate_path_segment("role-123_abc", "test").is_ok()); + assert!(validate_path_segment("bucket.with.dots", "test").is_ok()); + } +} From ee982d48754f2d511fbf6855a2f2853a9ebc8148 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 09:41:57 -0500 Subject: [PATCH 30/82] fix: Presigned URLs in debug logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed all four tracing::debug! calls in dispatch_operation from url = %fwd.url (which logged the full presigned URL including auth signatures in query params) to path = fwd.url.path() (which logs only the URL path — bucket and key, no credentials). The multipart backend_url log on line 418 was left as-is since that URL doesn't contain presigned auth params. --- crates/libs/core/src/proxy.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index feb2444..60cfe11 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -238,7 +238,7 @@ where ], ) .await?; - tracing::debug!(url = %fwd.url, "GET via presigned URL"); + tracing::debug!(path = fwd.url.path(), "GET via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::HeadObject { key, .. } => { @@ -256,7 +256,7 @@ where ], ) .await?; - tracing::debug!(url = %fwd.url, "HEAD via presigned URL"); + tracing::debug!(path = fwd.url.path(), "HEAD via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::PutObject { key, .. } => { @@ -269,14 +269,14 @@ where &["content-type", "content-length", "content-md5"], ) .await?; - tracing::debug!(url = %fwd.url, "PUT via presigned URL"); + tracing::debug!(path = fwd.url.path(), "PUT via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::DeleteObject { key, .. } => { let fwd = self .build_forward(Method::DELETE, bucket_config, key, original_headers, &[]) .await?; - tracing::debug!(url = %fwd.url, "DELETE via presigned URL"); + tracing::debug!(path = fwd.url.path(), "DELETE via presigned URL"); Ok(HandlerAction::Forward(fwd)) } S3Operation::ListBucket { raw_query, .. } => { From 8e9816516e64f3e14850834a4a7bcc7ad676d7b1 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 11:06:57 -0500 Subject: [PATCH 31/82] fix: Internal error details leaked to clients Determine level of log infomration by debug setting --- crates/libs/core/src/error.rs | 15 ++++++++++++ crates/libs/core/src/proxy.rs | 37 +++++++++++++++++++++++++---- crates/libs/core/src/s3/response.rs | 19 +++++++++++++-- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/crates/libs/core/src/error.rs b/crates/libs/core/src/error.rs index 1d6305a..cfd1a01 100644 --- a/crates/libs/core/src/error.rs +++ b/crates/libs/core/src/error.rs @@ -83,6 +83,21 @@ impl ProxyError { } } + /// Return a message safe to show to external clients. + /// + /// For server-side errors (500), returns a generic message to avoid + /// leaking backend infrastructure details. For client errors (4xx), + /// returns the full message (the client already knows the bucket name, + /// key, etc.). + pub fn safe_message(&self) -> String { + match self { + Self::BackendError(_) | Self::ConfigError(_) | Self::Internal(_) => { + "Internal server error".to_string() + } + other => other.to_string(), + } + } + /// Convert an `object_store::Error` into a `ProxyError`. pub fn from_object_store_error(e: object_store::Error) -> Self { match e { diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 60cfe11..e4f8768 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -70,6 +70,9 @@ pub struct PendingRequest { pub struct ProxyHandler { backend: B, resolver: R, + /// When true, error responses include full internal details (for development). + /// When false, server-side errors use generic messages. + debug_errors: bool, } impl ProxyHandler @@ -78,7 +81,21 @@ where R: RequestResolver, { pub fn new(backend: B, resolver: R) -> Self { - Self { backend, resolver } + Self { + backend, + resolver, + debug_errors: false, + } + } + + /// Enable verbose error messages in S3 error responses. + /// + /// When enabled, 500-class errors include the full internal message + /// (backend errors, config errors, etc.). Disable in production to + /// avoid leaking infrastructure details to clients. + pub fn with_debug_errors(mut self, enabled: bool) -> Self { + self.debug_errors = enabled; + self } /// Phase 1: Resolve an incoming request into an action. @@ -143,7 +160,12 @@ where s3_code = %err.s3_error_code(), "request failed" ); - HandlerAction::Response(error_response(&err, path, &request_id)) + HandlerAction::Response(error_response( + &err, + path, + &request_id, + self.debug_errors, + )) } } } @@ -169,7 +191,12 @@ where s3_code = %err.s3_error_code(), "multipart request failed" ); - error_response(&err, pending.operation.key(), &pending.request_id) + error_response( + &err, + pending.operation.key(), + &pending.request_id, + self.debug_errors, + ) } } } @@ -478,8 +505,8 @@ pub const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ "location", ]; -fn error_response(err: &ProxyError, resource: &str, request_id: &str) -> ProxyResult { - let xml = ErrorResponse::from_proxy_error(err, resource, request_id).to_xml(); +fn error_response(err: &ProxyError, resource: &str, request_id: &str, debug: bool) -> ProxyResult { + let xml = ErrorResponse::from_proxy_error(err, resource, request_id, debug).to_xml(); let body = ProxyResponseBody::from_bytes(Bytes::from(xml)); let mut headers = HeaderMap::new(); headers.insert("content-type", "application/xml".parse().unwrap()); diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs index 52608eb..f2f8405 100644 --- a/crates/libs/core/src/s3/response.rs +++ b/crates/libs/core/src/s3/response.rs @@ -20,10 +20,25 @@ pub struct ErrorResponse { } impl ErrorResponse { - pub fn from_proxy_error(err: &ProxyError, resource: &str, request_id: &str) -> Self { + /// Build an S3-compatible error response. + /// + /// When `debug` is `true`, the full internal error message is included + /// (useful during development). When `false`, server-side errors (500) + /// use a generic message to avoid leaking backend details. + pub fn from_proxy_error( + err: &ProxyError, + resource: &str, + request_id: &str, + debug: bool, + ) -> Self { + let message = if debug { + err.to_string() + } else { + err.safe_message() + }; Self { code: err.s3_error_code().to_string(), - message: err.to_string(), + message, resource: resource.to_string(), request_id: request_id.to_string(), } From e93b5b672b68da939745cfda7787252c77b42076 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 13:40:11 -0500 Subject: [PATCH 32/82] fix: Prefix auth lacks path boundary check Replaced key.starts_with(prefix) with key_matches_prefix(&key, prefix). The new function enforces a path boundary: if a prefix doesn't already end with /, the key must either equal the prefix exactly or have / as the next character. This prevents a scope prefix like data from granting access to data-private/secret.txt. --- crates/libs/core/src/auth.rs | 49 +++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index e02afd0..73bbf25 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -727,6 +727,50 @@ mod tests { assert!(matches!(err, ProxyError::AccessDenied)); }); } + + // ── Prefix boundary tests ───────────────────────────────────── + + #[test] + fn prefix_with_slash_matches_children() { + assert!(key_matches_prefix("data/file.txt", "data/")); + assert!(key_matches_prefix("data/sub/file.txt", "data/")); + } + + #[test] + fn prefix_without_slash_enforces_boundary() { + // Should match exact or with / boundary + assert!(key_matches_prefix("data/file.txt", "data")); + assert!(key_matches_prefix("data", "data")); + // Should NOT match sibling paths + assert!(!key_matches_prefix("data-private/secret.txt", "data")); + assert!(!key_matches_prefix("database/dump.sql", "data")); + } + + #[test] + fn empty_prefix_matches_everything() { + assert!(key_matches_prefix("anything/at/all.txt", "")); + assert!(key_matches_prefix("", "")); + } + + #[test] + fn prefix_no_match() { + assert!(!key_matches_prefix("other/file.txt", "data/")); + assert!(!key_matches_prefix("other/file.txt", "data")); + } +} + +/// Check if a key falls under an authorized prefix. +/// +/// If the prefix already ends with `/`, a plain `starts_with` is sufficient. +/// Otherwise we require that the key either equals the prefix exactly or +/// that the character immediately after the prefix is `/`. This prevents +/// a prefix like `data` from matching `data-private/secret.txt`. +fn key_matches_prefix(key: &str, prefix: &str) -> bool { + if prefix.ends_with('/') || prefix.is_empty() { + return key.starts_with(prefix); + } + // Prefix does not end with '/' — require an exact match or a '/' boundary + key == prefix || key.starts_with(&format!("{}/", prefix)) } /// Check if a resolved identity is authorized to perform an operation. @@ -785,7 +829,10 @@ pub fn authorize( if scope.prefixes.is_empty() { return true; // Full bucket access } - scope.prefixes.iter().any(|prefix| key.starts_with(prefix)) + scope + .prefixes + .iter() + .any(|prefix| key_matches_prefix(&key, prefix)) }); if authorized { From f47da3d44f8d540cc96d516c2f7348c0024bea6d Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 14:13:16 -0500 Subject: [PATCH 33/82] fix: prevent logging sensitive backend options --- crates/libs/core/src/types.rs | 69 +++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/crates/libs/core/src/types.rs b/crates/libs/core/src/types.rs index 7da526b..4f4fe74 100644 --- a/crates/libs/core/src/types.rs +++ b/crates/libs/core/src/types.rs @@ -3,9 +3,10 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::fmt; /// Configuration for a virtual bucket exposed by the proxy. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct BucketConfig { /// The virtual bucket name exposed to clients. pub name: String, @@ -32,6 +33,40 @@ pub struct BucketConfig { pub backend_options: HashMap, } +/// Keys in `backend_options` that hold secret values. +const REDACTED_OPTION_KEYS: &[&str] = &[ + "secret_access_key", + "access_key", + "service_account_key", + "token", +]; + +impl fmt::Debug for BucketConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let redacted_opts: HashMap<&str, &str> = self + .backend_options + .iter() + .map(|(k, v)| { + let val = if REDACTED_OPTION_KEYS.contains(&k.as_str()) { + "[REDACTED]" + } else { + v.as_str() + }; + (k.as_str(), val) + }) + .collect(); + + f.debug_struct("BucketConfig") + .field("name", &self.name) + .field("backend_type", &self.backend_type) + .field("backend_prefix", &self.backend_prefix) + .field("anonymous_access", &self.anonymous_access) + .field("allowed_roles", &self.allowed_roles) + .field("backend_options", &redacted_opts) + .finish() + } +} + /// Known backend provider types. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendType { @@ -114,7 +149,7 @@ pub enum Action { } /// A long-lived access credential stored in the config backend. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct StoredCredential { pub access_key_id: String, /// This is the HMAC signing key, not stored in plaintext ideally. @@ -126,8 +161,22 @@ pub struct StoredCredential { pub enabled: bool, } +impl fmt::Debug for StoredCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoredCredential") + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"[REDACTED]") + .field("principal_name", &self.principal_name) + .field("allowed_scopes", &self.allowed_scopes) + .field("created_at", &self.created_at) + .field("expires_at", &self.expires_at) + .field("enabled", &self.enabled) + .finish() + } +} + /// Temporary credentials minted by the STS API. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct TemporaryCredentials { pub access_key_id: String, pub secret_access_key: String, @@ -138,6 +187,20 @@ pub struct TemporaryCredentials { pub source_identity: String, } +impl fmt::Debug for TemporaryCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TemporaryCredentials") + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"[REDACTED]") + .field("session_token", &"[REDACTED]") + .field("expiration", &self.expiration) + .field("allowed_scopes", &self.allowed_scopes) + .field("assumed_role_id", &self.assumed_role_id) + .field("source_identity", &self.source_identity) + .finish() + } +} + /// Represents the resolved identity after authentication. #[derive(Debug, Clone)] pub enum ResolvedIdentity { From 5f302f447e3d1466b9856415fcae931f5a15afc6 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 14:49:46 -0500 Subject: [PATCH 34/82] feat: support pagination --- crates/libs/core/src/proxy.rs | 27 ++- crates/libs/core/src/s3/mod.rs | 1 + crates/libs/core/src/s3/pagination.rs | 259 ++++++++++++++++++++++++++ crates/libs/core/src/s3/response.rs | 14 +- 4 files changed, 291 insertions(+), 10 deletions(-) create mode 100644 crates/libs/core/src/s3/pagination.rs diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index e4f8768..82aa0aa 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -18,6 +18,7 @@ use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLO use crate::error::ProxyError; use crate::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use crate::response_body::ProxyResponseBody; +use crate::s3::pagination::{self, paginate}; use crate::s3::response::{ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents}; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; @@ -422,7 +423,8 @@ where &list_result, config, list_rewrite, - ); + raw_query, + )?; let mut resp_headers = HeaderMap::new(); resp_headers.insert("content-type", "application/xml".parse().unwrap()); @@ -594,7 +596,8 @@ fn build_list_xml( list_result: &object_store::ListResult, config: &BucketConfig, list_rewrite: Option<&ListRewrite>, -) -> String { + raw_query: Option<&str>, +) -> Result { let backend_prefix = config .backend_prefix .as_deref() @@ -635,18 +638,24 @@ fn build_list_xml( }) .collect(); - ListBucketResult { + let params = pagination::parse_pagination_params(raw_query); + let page = paginate(contents, common_prefixes, ¶ms)?; + + Ok(ListBucketResult { xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", name: bucket_name.to_string(), prefix: client_prefix.to_string(), delimiter: delimiter.to_string(), - max_keys: 1000, - is_truncated: false, - key_count: contents.len() + common_prefixes.len(), - contents, - common_prefixes, + max_keys: params.max_keys, + is_truncated: page.is_truncated, + key_count: page.contents.len() + page.common_prefixes.len(), + start_after: params.start_after, + continuation_token: params.continuation_token, + next_continuation_token: page.next_continuation_token, + contents: page.contents, + common_prefixes: page.common_prefixes, } - .to_xml() + .to_xml()) } /// Apply strip/add prefix rewriting to a key or prefix value. diff --git a/crates/libs/core/src/s3/mod.rs b/crates/libs/core/src/s3/mod.rs index 6f27e96..9540299 100644 --- a/crates/libs/core/src/s3/mod.rs +++ b/crates/libs/core/src/s3/mod.rs @@ -1,3 +1,4 @@ pub mod list_rewrite; +pub mod pagination; pub mod request; pub mod response; diff --git a/crates/libs/core/src/s3/pagination.rs b/crates/libs/core/src/s3/pagination.rs new file mode 100644 index 0000000..0b56f8a --- /dev/null +++ b/crates/libs/core/src/s3/pagination.rs @@ -0,0 +1,259 @@ +//! S3 ListObjectsV2 pagination as a post-processing step. +//! +//! `object_store::list_with_delimiter()` always fetches all results. +//! This module applies `max-keys`, `continuation-token`, and `start-after` +//! filtering on the full result set to produce S3-compliant paginated responses. + +use base64::Engine; + +use crate::error::ProxyError; +use crate::s3::response::{ListCommonPrefix, ListContents}; + +const DEFAULT_MAX_KEYS: usize = 1000; +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +pub struct PaginationParams { + pub max_keys: usize, + pub continuation_token: Option, + pub start_after: Option, +} + +pub struct PaginatedList { + pub contents: Vec, + pub common_prefixes: Vec, + pub is_truncated: bool, + pub next_continuation_token: Option, +} + +/// Parse `max-keys`, `continuation-token`, and `start-after` from a query string. +pub fn parse_pagination_params(raw_query: Option<&str>) -> PaginationParams { + let pairs = url::form_urlencoded::parse(raw_query.unwrap_or("").as_bytes()); + let find = |name| pairs.clone().find(|(k, _)| k == name).map(|(_, v)| v.to_string()); + + PaginationParams { + max_keys: find("max-keys") + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_MAX_KEYS) + .min(DEFAULT_MAX_KEYS), + continuation_token: find("continuation-token"), + start_after: find("start-after"), + } +} + +enum Entry { + Object(ListContents), + Prefix(ListCommonPrefix), +} + +impl Entry { + fn key(&self) -> &str { + match self { + Self::Object(c) => &c.key, + Self::Prefix(p) => &p.prefix, + } + } +} + +/// Apply pagination to a full list of objects and common prefixes. +pub fn paginate( + contents: Vec, + common_prefixes: Vec, + params: &PaginationParams, +) -> Result { + // Decode continuation token (takes precedence over start-after per S3 spec) + let decoded_token = params + .continuation_token + .as_deref() + .map(|t| { + B64.decode(t) + .ok() + .and_then(|b| String::from_utf8(b).ok()) + .ok_or_else(|| ProxyError::InvalidRequest("invalid continuation token".into())) + }) + .transpose()?; + + let start_after = decoded_token.as_deref().or(params.start_after.as_deref()); + + // Merge into a single sorted list + let mut entries: Vec = contents + .into_iter() + .map(Entry::Object) + .chain(common_prefixes.into_iter().map(Entry::Prefix)) + .collect(); + entries.sort_by(|a, b| a.key().cmp(b.key())); + + // Filter by start-after, then take max_keys + 1 to detect truncation + let mut page: Vec = match start_after { + Some(s) => entries.into_iter().filter(|e| e.key() > s).collect(), + None => entries, + }; + + let is_truncated = page.len() > params.max_keys; + page.truncate(params.max_keys); + + let next_continuation_token = + if is_truncated { page.last().map(|e| B64.encode(e.key())) } else { None }; + + // Split back into contents and common_prefixes + let mut result_contents = Vec::new(); + let mut result_prefixes = Vec::new(); + for entry in page { + match entry { + Entry::Object(c) => result_contents.push(c), + Entry::Prefix(p) => result_prefixes.push(p), + } + } + + Ok(PaginatedList { + contents: result_contents, + common_prefixes: result_prefixes, + is_truncated, + next_continuation_token, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_contents(keys: &[&str]) -> Vec { + keys.iter() + .map(|k| ListContents { + key: k.to_string(), + last_modified: "2024-01-01T00:00:00.000Z".to_string(), + etag: "\"abc\"".to_string(), + size: 100, + storage_class: "STANDARD", + }) + .collect() + } + + fn make_prefixes(prefixes: &[&str]) -> Vec { + prefixes + .iter() + .map(|p| ListCommonPrefix { prefix: p.to_string() }) + .collect() + } + + #[test] + fn parse_defaults() { + let p = parse_pagination_params(None); + assert_eq!(p.max_keys, 1000); + assert!(p.continuation_token.is_none()); + assert!(p.start_after.is_none()); + } + + #[test] + fn parse_max_keys_clamped_to_1000() { + assert_eq!(parse_pagination_params(Some("max-keys=5")).max_keys, 5); + assert_eq!(parse_pagination_params(Some("max-keys=9999")).max_keys, 1000); + assert_eq!(parse_pagination_params(Some("max-keys=abc")).max_keys, 1000); + } + + #[test] + fn parse_all_params() { + let token = B64.encode("some-key"); + let q = format!("max-keys=2&continuation-token={}&start-after=aaa", token); + let p = parse_pagination_params(Some(&q)); + assert_eq!(p.max_keys, 2); + assert_eq!(p.continuation_token.as_deref(), Some(token.as_str())); + assert_eq!(p.start_after.as_deref(), Some("aaa")); + } + + #[test] + fn no_truncation() { + let r = paginate(make_contents(&["a", "b", "c"]), vec![], &PaginationParams { + max_keys: 1000, continuation_token: None, start_after: None, + }).unwrap(); + assert_eq!(r.contents.len(), 3); + assert!(!r.is_truncated); + assert!(r.next_continuation_token.is_none()); + } + + #[test] + fn truncation_and_token() { + let r = paginate(make_contents(&["a", "b", "c", "d", "e"]), vec![], &PaginationParams { + max_keys: 2, continuation_token: None, start_after: None, + }).unwrap(); + assert_eq!(r.contents.len(), 2); + assert!(r.is_truncated); + assert_eq!(r.contents[0].key, "a"); + assert_eq!(r.contents[1].key, "b"); + assert!(r.next_continuation_token.is_some()); + } + + #[test] + fn continuation_token_round_trip() { + let items = make_contents(&["a", "b", "c", "d", "e"]); + let mk = |token| PaginationParams { max_keys: 2, continuation_token: token, start_after: None }; + + let p1 = paginate(items.clone(), vec![], &mk(None)).unwrap(); + assert_eq!(p1.contents.iter().map(|c| &c.key).collect::>(), &["a", "b"]); + + let p2 = paginate(items.clone(), vec![], &mk(p1.next_continuation_token)).unwrap(); + assert_eq!(p2.contents.iter().map(|c| &c.key).collect::>(), &["c", "d"]); + + let p3 = paginate(items.clone(), vec![], &mk(p2.next_continuation_token)).unwrap(); + assert_eq!(p3.contents.iter().map(|c| &c.key).collect::>(), &["e"]); + assert!(!p3.is_truncated); + } + + #[test] + fn start_after() { + let r = paginate(make_contents(&["a", "b", "c", "d"]), vec![], &PaginationParams { + max_keys: 1000, continuation_token: None, start_after: Some("b".into()), + }).unwrap(); + assert_eq!(r.contents.iter().map(|c| &c.key).collect::>(), &["c", "d"]); + } + + #[test] + fn continuation_token_overrides_start_after() { + let r = paginate(make_contents(&["a", "b", "c", "d", "e"]), vec![], &PaginationParams { + max_keys: 1000, + continuation_token: Some(B64.encode("c")), + start_after: Some("a".into()), + }).unwrap(); + assert_eq!(r.contents.iter().map(|c| &c.key).collect::>(), &["d", "e"]); + } + + #[test] + fn interleaved_objects_and_prefixes() { + let r = paginate( + make_contents(&["a.txt", "c.txt"]), + make_prefixes(&["b/", "d/"]), + &PaginationParams { max_keys: 3, continuation_token: None, start_after: None }, + ).unwrap(); + assert_eq!(r.contents.len(), 2); + assert_eq!(r.common_prefixes.len(), 1); + assert_eq!(r.contents[0].key, "a.txt"); + assert_eq!(r.common_prefixes[0].prefix, "b/"); + assert_eq!(r.contents[1].key, "c.txt"); + assert!(r.is_truncated); + } + + #[test] + fn invalid_token_returns_error() { + let r = paginate(make_contents(&["a"]), vec![], &PaginationParams { + max_keys: 1000, continuation_token: Some("not-valid!!!".into()), start_after: None, + }); + assert!(r.is_err()); + } + + #[test] + fn max_keys_zero() { + let r = paginate(make_contents(&["a", "b"]), vec![], &PaginationParams { + max_keys: 0, continuation_token: None, start_after: None, + }).unwrap(); + assert!(r.contents.is_empty()); + assert!(r.is_truncated); + } + + #[test] + fn empty_input() { + let r = paginate(vec![], vec![], &PaginationParams { + max_keys: 1000, continuation_token: None, start_after: None, + }).unwrap(); + assert_eq!(r.contents.len(), 0); + assert!(!r.is_truncated); + } +} diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs index f2f8405..08d7c15 100644 --- a/crates/libs/core/src/s3/response.rs +++ b/crates/libs/core/src/s3/response.rs @@ -172,13 +172,19 @@ pub struct ListBucketResult { pub is_truncated: bool, #[serde(rename = "KeyCount")] pub key_count: usize, + #[serde(rename = "StartAfter", skip_serializing_if = "Option::is_none")] + pub start_after: Option, + #[serde(rename = "ContinuationToken", skip_serializing_if = "Option::is_none")] + pub continuation_token: Option, + #[serde(rename = "NextContinuationToken", skip_serializing_if = "Option::is_none")] + pub next_continuation_token: Option, #[serde(rename = "Contents", default)] pub contents: Vec, #[serde(rename = "CommonPrefixes", default)] pub common_prefixes: Vec, } -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct ListContents { #[serde(rename = "Key")] pub key: String, @@ -221,6 +227,9 @@ mod tests { max_keys: 1000, is_truncated: false, key_count: 1, + start_after: None, + continuation_token: None, + next_continuation_token: None, contents: vec![ListContents { key: "photos/image.jpg".to_string(), last_modified: "2024-01-01T00:00:00.000Z".to_string(), @@ -254,6 +263,9 @@ mod tests { max_keys: 1000, is_truncated: false, key_count: 0, + start_after: None, + continuation_token: None, + next_continuation_token: None, contents: vec![], common_prefixes: vec![], }; From 63f1172095e46d729e5146b4c2c8d1f2c4fa58c8 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 24 Feb 2026 15:43:39 -0500 Subject: [PATCH 35/82] feat: JWKS improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix 1: JWKS Caching (jwks.rs) Added JwksCache struct with configurable TTL, backed by Mutex. get_or_fetch() returns cached responses if fresh, otherwise fetches from the network. assume_role_with_web_identity now takes a &JwksCache parameter — the caller creates one at startup and reuses it across requests. Fix 2: Clock Skew Tolerance + nbf Check (jwks.rs) Added 60-second clock skew tolerance to the exp check (now > exp + 60). Added nbf (not-before) validation with the same tolerance (now < nbf - 60). Fix 3: Minimum Duration Validation (lib.rs) Changed .min(role.max_session_duration_secs) to .clamp(900, role.max_session_duration_secs) — enforcing the same 900-second minimum that AWS uses. Fix 4: Algorithm Check Before JWKS Fetch (lib.rs) Added an early alg != "RS256" check on the unverified JWT header, right after the issuer check and before the JWKS fetch. Tokens with "none", "HS256", or other algorithms are now rejected without any network call. --- crates/libs/sts/src/jwks.rs | 68 ++++++++++++++++++++++++++++++++++--- crates/libs/sts/src/lib.rs | 23 ++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index 12cf7a5..dbb4bee 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -1,5 +1,9 @@ //! JWKS fetching and JWT verification. +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + use base64::Engine; use rsa::pkcs1v15::VerifyingKey; use rsa::signature::Verifier; @@ -9,12 +13,12 @@ use s3_proxy_core::types::RoleConfig; use serde::Deserialize; use sha2::Sha256; -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct JwksResponse { pub keys: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct JwkKey { pub kid: String, pub kty: String, @@ -165,13 +169,67 @@ pub fn verify_token( } } - // Validate expiration + // Validate time-based claims with clock skew tolerance + let now = chrono::Utc::now().timestamp(); + const CLOCK_SKEW_SECS: i64 = 60; + if let Some(exp) = claims.get("exp").and_then(|v| v.as_i64()) { - let now = chrono::Utc::now().timestamp(); - if now > exp { + if now > exp + CLOCK_SKEW_SECS { return Err(ProxyError::InvalidOidcToken("token has expired".into())); } } + if let Some(nbf) = claims.get("nbf").and_then(|v| v.as_i64()) { + if now < nbf - CLOCK_SKEW_SECS { + return Err(ProxyError::InvalidOidcToken("token is not yet valid".into())); + } + } + Ok(claims) } + +/// In-memory cache for JWKS responses, keyed by issuer URL. +/// +/// OIDC providers publish JWKS keys that change infrequently. Caching avoids +/// a network round-trip to the provider on every token validation and prevents +/// DoS via repeated validation attempts. +pub struct JwksCache { + ttl: Duration, + entries: Mutex>, +} + +impl JwksCache { + /// Create a new cache with the given TTL. + pub fn new(ttl: Duration) -> Self { + Self { + ttl, + entries: Mutex::new(HashMap::new()), + } + } + + /// Fetch JWKS for the given issuer, returning a cached response if fresh. + pub async fn get_or_fetch(&self, issuer: &str) -> Result { + // Check cache + if let Some(cached) = self.get_cached(issuer) { + return Ok(cached); + } + + // Cache miss — fetch from the network + let jwks = fetch_jwks(issuer).await?; + + let mut entries = self.entries.lock().unwrap(); + entries.insert(issuer.to_string(), (Instant::now(), jwks.clone())); + + Ok(jwks) + } + + fn get_cached(&self, issuer: &str) -> Option { + let entries = self.entries.lock().unwrap(); + if let Some((fetched_at, jwks)) = entries.get(issuer) { + if fetched_at.elapsed() < self.ttl { + return Some(jwks.clone()); + } + } + None + } +} diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index 00415cd..20391a3 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -21,6 +21,7 @@ pub mod responses; pub mod sts; use base64::Engine; +pub use jwks::JwksCache; use request::StsRequest; use s3_proxy_core::config::ConfigProvider; use s3_proxy_core::error::ProxyError; @@ -54,6 +55,7 @@ pub async fn assume_role_with_web_identity( config: &C, sts_request: &StsRequest, key_prefix: &str, + jwks_cache: &JwksCache, ) -> Result { // Look up the role let role = config @@ -77,8 +79,20 @@ pub async fn assume_role_with_web_identity( ))); } - // Fetch JWKS and verify the token - let jwks = jwks::fetch_jwks(issuer).await?; + // Fail fast on unsupported algorithms before making any network requests + let alg = header + .get("alg") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if alg != "RS256" { + return Err(ProxyError::InvalidOidcToken(format!( + "unsupported JWT algorithm: {}", + alg + ))); + } + + // Fetch JWKS (using cache) and verify the token + let jwks = jwks_cache.get_or_fetch(issuer).await?; let kid = header .get("kid") .and_then(|v| v.as_str()) @@ -103,11 +117,12 @@ pub async fn assume_role_with_web_identity( } } - // Mint temporary credentials + // Mint temporary credentials (AWS enforces 900s minimum) + const MIN_SESSION_DURATION_SECS: u64 = 900; let duration = sts_request .duration_seconds .unwrap_or(3600) - .min(role.max_session_duration_secs); + .clamp(MIN_SESSION_DURATION_SECS, role.max_session_duration_secs); let creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix); From 5a65b71a7ddc1b422dd55351af9e33a6275c448a Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 15:16:08 -0500 Subject: [PATCH 36/82] feat: refactor to use Axom, wire in STS --- crates/libs/core/Cargo.toml | 4 + crates/libs/core/src/axum.rs | 33 ++ crates/libs/core/src/lib.rs | 2 + crates/libs/sts/src/jwks.rs | 30 +- crates/libs/sts/src/lib.rs | 25 ++ crates/libs/sts/src/responses.rs | 44 +++ crates/runtimes/cf-workers/Cargo.toml | 6 +- crates/runtimes/cf-workers/src/body.rs | 31 -- crates/runtimes/cf-workers/src/lib.rs | 339 ++++++++++----------- crates/runtimes/server/Cargo.toml | 8 +- crates/runtimes/server/src/bin/s3-proxy.rs | 19 +- crates/runtimes/server/src/body.rs | 41 --- crates/runtimes/server/src/lib.rs | 8 +- crates/runtimes/server/src/server.rs | 170 ++++++----- 14 files changed, 409 insertions(+), 351 deletions(-) create mode 100644 crates/libs/core/src/axum.rs delete mode 100644 crates/runtimes/cf-workers/src/body.rs delete mode 100644 crates/runtimes/server/src/body.rs diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml index 2dec1a8..171795d 100644 --- a/crates/libs/core/Cargo.toml +++ b/crates/libs/core/Cargo.toml @@ -7,6 +7,7 @@ description = "Runtime-agnostic core library for the S3 proxy gateway" [features] default = [] +axum = ["dep:axum"] config-dynamodb = ["aws-sdk-dynamodb", "tokio"] config-postgres = ["sqlx"] config-http = ["reqwest"] @@ -33,6 +34,9 @@ tracing.workspace = true object_store.workspace = true futures.workspace = true +# Optional framework deps +axum = { workspace = true, features = ["json"], optional = true } + # Optional config backend deps aws-sdk-dynamodb = { workspace = true, optional = true } tokio = { workspace = true, optional = true } diff --git a/crates/libs/core/src/axum.rs b/crates/libs/core/src/axum.rs new file mode 100644 index 0000000..49b4c62 --- /dev/null +++ b/crates/libs/core/src/axum.rs @@ -0,0 +1,33 @@ +//! Axum response helpers shared across runtimes. +//! +//! Gated behind the `axum` feature flag so the core crate remains usable +//! without pulling in axum. + +use ::axum::body::Body; +use ::axum::response::Response; + +use crate::proxy::ProxyResult; +use crate::response_body::ProxyResponseBody; + +/// Convert a [`ProxyResult`] to an axum [`Response`]. +pub fn build_proxy_response(result: ProxyResult) -> Response { + let body = match result.body { + ProxyResponseBody::Bytes(b) => Body::from(b), + ProxyResponseBody::Empty => Body::empty(), + }; + + let mut builder = Response::builder().status(result.status); + for (key, value) in result.headers.iter() { + builder = builder.header(key, value); + } + + builder.body(body).unwrap() +} + +/// Build a plain-text error response. +pub fn error_response(status: u16, message: &str) -> Response { + Response::builder() + .status(status) + .body(Body::from(message.to_string())) + .unwrap() +} diff --git a/crates/libs/core/src/lib.rs b/crates/libs/core/src/lib.rs index 478ce3f..4fcfe2e 100644 --- a/crates/libs/core/src/lib.rs +++ b/crates/libs/core/src/lib.rs @@ -17,6 +17,8 @@ //! - [`proxy::ProxyHandler`] — the main request handler that ties everything together pub mod auth; +#[cfg(feature = "axum")] +pub mod axum; pub mod backend; pub mod config; pub mod error; diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index dbb4bee..4992335 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use std::sync::Mutex; -use std::time::{Duration, Instant}; +use std::time::Duration; + +use chrono::{DateTime, Utc}; use base64::Engine; use rsa::pkcs1v15::VerifyingKey; @@ -30,12 +32,14 @@ pub struct JwkKey { } /// Fetch JWKS from an OIDC provider's well-known endpoint. -pub async fn fetch_jwks(issuer: &str) -> Result { +pub async fn fetch_jwks( + client: &reqwest::Client, + issuer: &str, +) -> Result { let issuer = issuer.trim_end_matches('/'); // First, try the .well-known/openid-configuration endpoint let config_url = format!("{}/.well-known/openid-configuration", issuer); - let client = reqwest::Client::new(); let config_resp = client.get(&config_url).send().await.map_err(|e| { @@ -193,15 +197,20 @@ pub fn verify_token( /// OIDC providers publish JWKS keys that change infrequently. Caching avoids /// a network round-trip to the provider on every token validation and prevents /// DoS via repeated validation attempts. +/// +/// Uses `DateTime` instead of `std::time::Instant` for WASM compatibility +/// (`Instant` panics on `wasm32-unknown-unknown`). pub struct JwksCache { + client: reqwest::Client, ttl: Duration, - entries: Mutex>, + entries: Mutex, JwksResponse)>>, } impl JwksCache { - /// Create a new cache with the given TTL. - pub fn new(ttl: Duration) -> Self { + /// Create a new cache with the given TTL and HTTP client. + pub fn new(client: reqwest::Client, ttl: Duration) -> Self { Self { + client, ttl, entries: Mutex::new(HashMap::new()), } @@ -215,10 +224,10 @@ impl JwksCache { } // Cache miss — fetch from the network - let jwks = fetch_jwks(issuer).await?; + let jwks = fetch_jwks(&self.client, issuer).await?; let mut entries = self.entries.lock().unwrap(); - entries.insert(issuer.to_string(), (Instant::now(), jwks.clone())); + entries.insert(issuer.to_string(), (Utc::now(), jwks.clone())); Ok(jwks) } @@ -226,7 +235,10 @@ impl JwksCache { fn get_cached(&self, issuer: &str) -> Option { let entries = self.entries.lock().unwrap(); if let Some((fetched_at, jwks)) = entries.get(issuer) { - if fetched_at.elapsed() < self.ttl { + let elapsed = Utc::now() + .signed_duration_since(*fetched_at) + .num_seconds(); + if elapsed >= 0 && (elapsed as u64) < self.ttl.as_secs() { return Some(jwks.clone()); } } diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index 20391a3..6ea54b7 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -22,11 +22,36 @@ pub mod sts; use base64::Engine; pub use jwks::JwksCache; +pub use request::try_parse_sts_request; use request::StsRequest; +pub use responses::{build_sts_error_response, build_sts_response}; use s3_proxy_core::config::ConfigProvider; use s3_proxy_core::error::ProxyError; use s3_proxy_core::types::TemporaryCredentials; +/// Try to handle an STS request. Returns `Some((status, xml))` if the query +/// contained an STS action, or `None` if it wasn't an STS request. +pub async fn try_handle_sts( + query: Option<&str>, + config: &C, + jwks_cache: &JwksCache, +) -> Option<(u16, String)> { + let sts_result = try_parse_sts_request(query)?; + let (status, xml) = match sts_result { + Ok(sts_request) => { + match assume_role_with_web_identity(config, &sts_request, "STSPRXY", jwks_cache).await { + Ok(creds) => build_sts_response(&creds), + Err(e) => { + tracing::warn!(error = %e, "STS request failed"); + build_sts_error_response(&e) + } + } + } + Err(e) => build_sts_error_response(&e), + }; + Some((status, xml)) +} + /// Decode JWT header and claims without signature verification. fn jwt_decode_unverified( token: &str, diff --git a/crates/libs/sts/src/responses.rs b/crates/libs/sts/src/responses.rs index 601dffa..a8d2e40 100644 --- a/crates/libs/sts/src/responses.rs +++ b/crates/libs/sts/src/responses.rs @@ -1,6 +1,8 @@ //! STS XML response serialization. use quick_xml::se::to_string as xml_to_string; +use s3_proxy_core::error::ProxyError; +use s3_proxy_core::types::TemporaryCredentials; use serde::Serialize; /// STS AssumeRoleWithWebIdentity response. @@ -47,3 +49,45 @@ impl AssumeRoleWithWebIdentityResponse { ) } } + +/// Build an STS success response (status code + XML body) from temporary credentials. +pub fn build_sts_response(creds: &TemporaryCredentials) -> (u16, String) { + let response = AssumeRoleWithWebIdentityResponse { + result: AssumeRoleWithWebIdentityResult { + credentials: StsCredentials { + access_key_id: creds.access_key_id.clone(), + secret_access_key: creds.secret_access_key.clone(), + session_token: creds.session_token.clone(), + expiration: creds.expiration.to_rfc3339(), + }, + assumed_role_user: AssumedRoleUser { + assumed_role_id: creds.assumed_role_id.clone(), + arn: creds.assumed_role_id.clone(), + }, + }, + }; + (200, response.to_xml()) +} + +/// Build an STS error response (status code + XML body) from a ProxyError. +pub fn build_sts_error_response(err: &ProxyError) -> (u16, String) { + let (status, code, message) = match err { + ProxyError::RoleNotFound(r) => (400, "MalformedPolicyDocument", format!("role not found: {}", r)), + ProxyError::InvalidOidcToken(msg) => (400, "InvalidIdentityToken", msg.clone()), + ProxyError::InvalidRequest(msg) => (400, "InvalidParameterValue", msg.clone()), + ProxyError::AccessDenied => (403, "AccessDenied", "access denied".to_string()), + _ => (500, "InternalError", "internal error".to_string()), + }; + + let xml = format!( + "\n\ + \ + \ + {}\ + {}\ + \ + ", + code, message + ); + (status, xml) +} diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index 3571c5d..1b97110 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -9,9 +9,10 @@ description = "Cloudflare Workers runtime for the S3 proxy gateway" crate-type = ["cdylib"] [dependencies] -s3-proxy-core = { workspace = true, features = [] } +s3-proxy-core = { workspace = true, features = ["axum"] } s3-proxy-sts.workspace = true s3-proxy-source-coop.workspace = true +axum = { workspace = true, features = ["json"] } bytes.workspace = true http.workspace = true serde.workspace = true @@ -26,9 +27,10 @@ futures.workspace = true http-body.workspace = true http-body-util.workspace = true async-trait.workspace = true +reqwest.workspace = true # Cloudflare Workers SDK -worker = "0.7" +worker = { version = "0.7", features = ["http", "axum"] } wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" js-sys = "0.3" diff --git a/crates/runtimes/cf-workers/src/body.rs b/crates/runtimes/cf-workers/src/body.rs deleted file mode 100644 index c8383a8..0000000 --- a/crates/runtimes/cf-workers/src/body.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Response body conversion for the Cloudflare Workers runtime. -//! -//! Converts [`ProxyResult`] to `worker::Response`. Only handles non-streaming -//! bodies (Bytes, Empty). Streaming responses go through the Forward path -//! in `lib.rs`, which uses the Fetch API directly. - -use s3_proxy_core::proxy::ProxyResult; -use s3_proxy_core::response_body::ProxyResponseBody; -use worker::{Headers, Response}; - -/// Build a `worker::Response` from a `ProxyResult`. -/// -/// Only handles `Bytes` and `Empty` bodies (LIST XML, errors, multipart XML). -/// Streaming Forward responses are built directly in `lib.rs`. -pub fn build_worker_response(result: ProxyResult) -> Result { - let resp_headers = Headers::new(); - for (key, value) in result.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = resp_headers.set(key.as_str(), v); - } - } - - match result.body { - ProxyResponseBody::Bytes(b) => Ok(Response::from_bytes(b.to_vec())? - .with_status(result.status) - .with_headers(resp_headers)), - ProxyResponseBody::Empty => Ok(Response::from_bytes(vec![])? - .with_status(result.status) - .with_headers(resp_headers)), - } -} diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index e55f633..16c6c37 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -1,18 +1,16 @@ //! Cloudflare Workers runtime for the S3 proxy gateway. //! //! This crate provides implementations of core traits using Cloudflare Workers -//! primitives. Uses a two-phase request handling model: -//! -//! 1. **`resolve_request`** determines the action (Forward, Response, NeedsBody) -//! 2. **Forward** requests execute presigned URLs via the Fetch API, passing -//! JS `ReadableStream` bodies directly — zero Rust stream involvement. +//! primitives. Uses the worker crate's `http` feature for standard +//! `http::Request`/`http::Response` types, eliminating manual type conversion. +//! Uses reqwest (which wraps `web_sys::fetch` on WASM) for forward execution. //! //! # Architecture //! //! ```text -//! Client -> Worker (JS Request) +//! Client -> Worker (http::Request via worker's `http` feature) //! -> resolve request (core resolver or Source Cooperative resolver) -//! -> Forward: fetch(presigned URL) with ReadableStream passthrough +//! -> Forward: reqwest with presigned URL //! -> Response: LIST XML via object_store, errors, synthetic responses //! -> NeedsBody: multipart operations via raw signed HTTP //! ``` @@ -25,39 +23,71 @@ //! - The HTTP config provider for centralized config APIs //! - **Source Cooperative API** when `SOURCE_API_URL` is set -mod body; mod client; mod fetch_connector; mod tracing_layer; -use body::build_worker_response; -use client::{extract_response_headers, WorkerBackend}; +use client::WorkerBackend; +use s3_proxy_core::axum::{build_proxy_response, error_response}; use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; -use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler}; +use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; use s3_proxy_core::resolver::{DefaultResolver, RequestResolver}; use s3_proxy_source_coop::api::{CacheTtls, SourceApiClient}; use s3_proxy_source_coop::resolver::SourceCoopResolver; +use s3_proxy_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; + +use axum::body::Body; +use axum::response::Response; +use http::HeaderMap; use worker::*; /// The Worker entry point. /// +/// With the `http` feature, the worker crate provides standard `http::Request` +/// and `http::Response` types, eliminating the need for manual method/header +/// conversion. +/// /// Wrangler config (`wrangler.toml`) should bind: /// - `CONFIG` environment variable or KV namespace for configuration /// - `VIRTUAL_HOST_DOMAIN` environment variable (optional) /// - `SOURCE_API_URL` + `SOURCE_API_KEY` for Source Cooperative API mode #[event(fetch)] -async fn main(req: Request, env: Env, _ctx: Context) -> Result { +async fn fetch( + req: HttpRequest, + env: Env, + _ctx: Context, +) -> Result> { // Initialize panic hook for better error messages console_error_panic_hook::set_once(); // Initialize tracing subscriber (idempotent — ignored if already set) tracing::subscriber::set_global_default(tracing_layer::WorkerSubscriber::new()).ok(); - let method = convert_method(&req); - let url = req.url()?; - let path = url.path().to_string(); - let query = url.query().map(|q| q.to_string()); - let headers = convert_headers(&req); + let reqwest_client = reqwest::Client::new(); + let jwks_cache = JwksCache::new(reqwest_client.clone(), std::time::Duration::from_secs(900)); + + let (parts, worker_body) = req.into_parts(); + let body = Body::new(worker_body); + let method = parts.method; + let uri = parts.uri; + let path = uri.path().to_string(); + let query = uri.query().map(|q| q.to_string()); + let headers = parts.headers; + + // Intercept STS AssumeRoleWithWebIdentity requests before resolver dispatch. + // STS uses STS_CONFIG (falling back to PROXY_CONFIG) for role definitions. + if try_parse_sts_request(query.as_deref()).is_some() { + let config = load_sts_config(&env)?; + if let Some((status, xml)) = + try_handle_sts(query.as_deref(), &config, &jwks_cache).await + { + return Ok(Response::builder() + .status(status) + .header("content-type", "application/xml") + .body(Body::from(xml)) + .unwrap()); + } + } // Source Cooperative API mode: when SOURCE_API_URL is set, resolve backends // dynamically from the Source API instead of static PROXY_CONFIG. @@ -77,32 +107,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { "SOURCE_API_URL set, using Source Cooperative API resolver" ); - let mut cache_ttls = CacheTtls::default(); - if let Ok(v) = env.var("SOURCE_CACHE_TTL_PRODUCT") { - if let Ok(n) = v.to_string().parse::() { - cache_ttls.product = n; - } - } - if let Ok(v) = env.var("SOURCE_CACHE_TTL_DATA_CONNECTION") { - if let Ok(n) = v.to_string().parse::() { - cache_ttls.data_connection = n; - } - } - if let Ok(v) = env.var("SOURCE_CACHE_TTL_PERMISSIONS") { - if let Ok(n) = v.to_string().parse::() { - cache_ttls.permissions = n; - } - } - if let Ok(v) = env.var("SOURCE_CACHE_TTL_ACCOUNT") { - if let Ok(n) = v.to_string().parse::() { - cache_ttls.account = n; - } - } - if let Ok(v) = env.var("SOURCE_CACHE_TTL_API_KEY") { - if let Ok(n) = v.to_string().parse::() { - cache_ttls.api_key = n; - } - } + let cache_ttls = load_cache_ttls(&env); let api_client = SourceApiClient::new( client::WorkerHttpClient, @@ -113,185 +118,173 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let resolver = SourceCoopResolver::new(api_client); let handler = ProxyHandler::new(WorkerBackend, resolver); - return handle_action(&req, method, &handler, &path, query.as_deref(), &headers).await; - } - - // Load PROXY_CONFIG from environment. - // Supports two formats: - // - JSON string (e.g., set via `wrangler secret` or a plain string var) - // - JS object (e.g., set via `[vars.PROXY_CONFIG]` table in wrangler.toml) - let config = if let Ok(var) = env.var("PROXY_CONFIG") { - let config_str = var.to_string(); - tracing::debug!( - config_len = config_str.len(), - "loaded PROXY_CONFIG as string" + return Ok( + handle_action(method, &handler, &reqwest_client, &path, query.as_deref(), &headers, body) + .await, ); - StaticProvider::from_json(&config_str) - .map_err(|e| worker::Error::RustError(format!("config error: {}", e)))? - } else { - tracing::debug!("loading PROXY_CONFIG as object"); - let static_config: StaticConfig = env - .object_var("PROXY_CONFIG") - .map_err(|e| worker::Error::RustError(format!("config error: {}", e)))?; - StaticProvider::from_config(static_config) - }; + } + let config = load_static_config(&env)?; let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); let resolver = DefaultResolver::new(config, virtual_host_domain); let handler = ProxyHandler::new(WorkerBackend, resolver); - handle_action(&req, method, &handler, &path, query.as_deref(), &headers).await + Ok( + handle_action(method, &handler, &reqwest_client, &path, query.as_deref(), &headers, body) + .await, + ) } // ── Two-phase request handling ────────────────────────────────────── /// Handle the resolved action for any resolver type. async fn handle_action( - req: &Request, method: http::Method, handler: &ProxyHandler, + client: &reqwest::Client, path: &str, query: Option<&str>, headers: &http::HeaderMap, -) -> Result { + body: Body, +) -> Response { let action = handler.resolve_request(method, path, query, headers).await; match action { - HandlerAction::Response(result) => build_worker_response(result), - HandlerAction::Forward(fwd) => execute_forward(req, fwd).await, + HandlerAction::Response(result) => build_proxy_response(result), + HandlerAction::Forward(fwd) => forward_to_backend(client, fwd, body).await, HandlerAction::NeedsBody(pending) => { - let body = read_request_body(req).await?; - let result = handler.handle_with_body(pending, body).await; - build_worker_response(result) + let collected = match axum::body::to_bytes(body, usize::MAX).await { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "failed to read request body"); + return error_response(500, "Internal error"); + } + }; + let result = handler.handle_with_body(pending, collected).await; + build_proxy_response(result) } } } -/// Execute a Forward request via the Fetch API. +/// Execute a Forward request via reqwest. /// -/// For PUT: passes the original JS `ReadableStream` body directly to fetch. -/// For GET: returns the response `ReadableStream` directly to the client. -/// Zero Rust stream involvement — bytes never cross the WASM boundary. -async fn execute_forward(req: &Request, fwd: ForwardRequest) -> Result { - // Build request headers - let ws_headers = web_sys::Headers::new() - .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; - - for (key, value) in fwd.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } +/// On WASM, reqwest wraps `web_sys::fetch` internally. Bodies are collected +/// to bytes since WASM reqwest doesn't support streaming. +async fn forward_to_backend( + client: &reqwest::Client, + fwd: ForwardRequest, + body: Body, +) -> Response { + let mut req_builder = client.request(fwd.method.clone(), fwd.url.as_str()); + + for (k, v) in fwd.headers.iter() { + req_builder = req_builder.header(k, v); } - // Build fetch request - let init = web_sys::RequestInit::new(); - init.set_method(fwd.method.as_str()); - init.set_headers(&ws_headers.into()); - - // For PUT: pass original request body stream directly + // Attach body for PUT — collect to bytes since WASM reqwest + // doesn't support wrap_stream if fwd.method == http::Method::PUT { - if let Some(body_stream) = req.inner().body() { - init.set_body(&body_stream.into()); + match axum::body::to_bytes(body, usize::MAX).await { + Ok(bytes) => { + req_builder = req_builder.body(bytes); + } + Err(e) => { + tracing::error!(error = %e, "failed to read PUT body"); + return error_response(500, "Internal error"); + } } } - let ws_request = web_sys::Request::new_with_str_and_init(fwd.url.as_str(), &init) - .map_err(|e| worker::Error::RustError(format!("request error: {:?}", e)))?; - - // Execute fetch - let worker_req: worker::Request = ws_request.into(); - let worker_resp = Fetch::Request(worker_req) - .send() - .await - .map_err(|e| worker::Error::RustError(format!("forward fetch failed: {}", e)))?; + let backend_resp = match req_builder.send().await { + Ok(resp) => resp, + Err(e) => { + tracing::error!(error = %e, "forward request failed"); + return error_response(502, "Bad Gateway"); + } + }; - let status = worker_resp.status_code(); - let ws_response: web_sys::Response = worker_resp.into(); + let status = backend_resp.status().as_u16(); // Forward allowlisted response headers - let resp_headers = extract_response_headers(&ws_response.headers()); - let ws_resp_headers = web_sys::Headers::new() - .map_err(|e| worker::Error::RustError(format!("headers error: {:?}", e)))?; - for (key, value) in resp_headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_resp_headers.set(key.as_str(), v); + let mut resp_headers = HeaderMap::new(); + for name in RESPONSE_HEADER_ALLOWLIST { + if let Some(v) = backend_resp.headers().get(*name) { + resp_headers.insert(*name, v.clone()); } } - // Build response with the backend's ReadableStream body (passthrough) - let resp_init = web_sys::ResponseInit::new(); - resp_init.set_status(status); - resp_init.set_headers(&ws_resp_headers.into()); + // Read response body as bytes (WASM reqwest doesn't support bytes_stream) + let resp_bytes = match backend_resp.bytes().await { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "failed to read backend response"); + return error_response(502, "Bad Gateway"); + } + }; - let body = ws_response.body(); - let response = - web_sys::Response::new_with_opt_readable_stream_and_init(body.as_ref(), &resp_init) - .map_err(|e| worker::Error::RustError(format!("response error: {:?}", e)))?; + let mut builder = Response::builder().status(status); + for (k, v) in resp_headers.iter() { + builder = builder.header(k, v); + } - Ok(response.into()) + builder.body(Body::from(resp_bytes)).unwrap() } // ── Shared helpers ────────────────────────────────────────────────── -/// Read a Worker request body into Bytes. -async fn read_request_body(req: &Request) -> Result { - // Extract body as ReadableStream, consume to bytes - let ws_request = req.inner(); - match ws_request.body() { - Some(stream) => { - let response = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) - .map_err(|e| worker::Error::RustError(format!("failed to wrap stream: {:?}", e)))?; - - let array_buffer_promise = response.array_buffer().map_err(|e| { - worker::Error::RustError(format!("failed to get arrayBuffer: {:?}", e)) - })?; - - let array_buffer = wasm_bindgen_futures::JsFuture::from(array_buffer_promise) - .await - .map_err(|e| { - worker::Error::RustError(format!("failed to read arrayBuffer: {:?}", e)) - })?; - - let uint8 = js_sys::Uint8Array::new(&array_buffer); - Ok(bytes::Bytes::from(uint8.to_vec())) - } - None => Ok(bytes::Bytes::new()), +/// Load a StaticProvider from a named env var (supports both JSON string and JS object). +fn load_config_from_env(env: &Env, var_name: &str) -> Result { + if let Ok(var) = env.var(var_name) { + let config_str = var.to_string(); + tracing::debug!(var = var_name, config_len = config_str.len(), "loaded config as string"); + StaticProvider::from_json(&config_str) + .map_err(|e| worker::Error::RustError(format!("{} config error: {}", var_name, e))) + } else { + tracing::debug!(var = var_name, "loading config as object"); + let static_config: StaticConfig = env + .object_var(var_name) + .map_err(|e| worker::Error::RustError(format!("{} config error: {}", var_name, e)))?; + Ok(StaticProvider::from_config(static_config)) } } -fn convert_method(req: &Request) -> http::Method { - match req.method() { - Method::Get => http::Method::GET, - Method::Head => http::Method::HEAD, - Method::Post => http::Method::POST, - Method::Put => http::Method::PUT, - Method::Delete => http::Method::DELETE, - _ => http::Method::GET, - } +fn load_static_config(env: &Env) -> Result { + load_config_from_env(env, "PROXY_CONFIG") } -fn convert_headers(req: &Request) -> http::HeaderMap { - let mut headers = http::HeaderMap::new(); - for name in &[ - "authorization", - "host", - "x-amz-date", - "x-amz-content-sha256", - "x-amz-security-token", - "content-type", - "content-length", - "content-md5", - "range", - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", - ] { - if let Ok(Some(value)) = req.headers().get(name) { - if let Ok(parsed) = value.parse() { - headers.insert(*name, parsed); - } +/// Load STS config: tries STS_CONFIG first, falls back to PROXY_CONFIG. +fn load_sts_config(env: &Env) -> Result { + load_config_from_env(env, "STS_CONFIG") + .or_else(|_| load_config_from_env(env, "PROXY_CONFIG")) +} + +/// Load cache TTL overrides from environment variables. +fn load_cache_ttls(env: &Env) -> CacheTtls { + let mut cache_ttls = CacheTtls::default(); + if let Ok(v) = env.var("SOURCE_CACHE_TTL_PRODUCT") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.product = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_DATA_CONNECTION") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.data_connection = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_PERMISSIONS") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.permissions = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_ACCOUNT") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.account = n; + } + } + if let Ok(v) = env.var("SOURCE_CACHE_TTL_API_KEY") { + if let Ok(n) = v.to_string().parse::() { + cache_ttls.api_key = n; } } - headers + cache_ttls } diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index a7b12ca..9427ffa 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -6,13 +6,12 @@ license.workspace = true description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] -s3-proxy-core = { workspace = true, features = ["azure", "gcp"] } +s3-proxy-core = { workspace = true, features = ["axum", "azure", "gcp"] } s3-proxy-sts.workspace = true s3-proxy-source-coop.workspace = true +axum = { workspace = true, features = ["json", "tokio", "http1", "http2"] } +tower-service.workspace = true tokio.workspace = true -hyper.workspace = true -hyper-util.workspace = true -http-body-util.workspace = true http.workspace = true bytes.workspace = true tracing.workspace = true @@ -23,3 +22,4 @@ reqwest = { workspace = true, features = ["stream"] } thiserror.workspace = true object_store.workspace = true futures.workspace = true +http-body-util.workspace = true diff --git a/crates/runtimes/server/src/bin/s3-proxy.rs b/crates/runtimes/server/src/bin/s3-proxy.rs index 9f0dcd2..1a6b75a 100644 --- a/crates/runtimes/server/src/bin/s3-proxy.rs +++ b/crates/runtimes/server/src/bin/s3-proxy.rs @@ -1,7 +1,7 @@ //! S3 Proxy Server binary. //! //! Usage: -//! s3-proxy --config config.toml [--listen 0.0.0.0:8080] [--domain s3.local] +//! s3-proxy --config config.toml [--sts-config sts.toml] [--listen 0.0.0.0:8080] [--domain s3.local] use s3_proxy_core::config::cached::CachedProvider; use s3_proxy_core::config::static_file::StaticProvider; @@ -40,15 +40,30 @@ async fn main() -> Result<(), Box> { .and_then(|i| args.get(i + 1)) .cloned(); + let sts_config_path = args + .iter() + .position(|a| a == "--sts-config") + .and_then(|i| args.get(i + 1)) + .map(|s| s.as_str()); + tracing::info!(config = %config_path, listen = %listen_addr, "starting s3-proxy"); let base_config = StaticProvider::from_file(config_path)?; + let sts_base = match sts_config_path { + Some(path) => { + tracing::info!(sts_config = %path, "using separate STS config"); + StaticProvider::from_file(path)? + } + None => base_config.clone(), + }; + let config = CachedProvider::new(base_config, Duration::from_secs(60)); + let sts_config = CachedProvider::new(sts_base, Duration::from_secs(60)); let server_config = ServerConfig { listen_addr, virtual_host_domain: domain, }; - run(config, server_config).await + run(config, sts_config, server_config).await } diff --git a/crates/runtimes/server/src/body.rs b/crates/runtimes/server/src/body.rs deleted file mode 100644 index b369f55..0000000 --- a/crates/runtimes/server/src/body.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Response body conversion for the server runtime. -//! -//! Converts [`ProxyResult`] to a hyper response body. Streaming responses -//! (from Forward requests) are handled directly in `server.rs`. - -use bytes::Bytes; -use futures::Stream; -use http::Response; -use http_body_util::{Either, Empty, Full, StreamBody}; -use hyper::body::Frame; -use s3_proxy_core::proxy::ProxyResult; -use s3_proxy_core::response_body::ProxyResponseBody; -use std::pin::Pin; - -/// A boxed streaming body type that erases concrete stream types. -type BoxedStreamBody = - StreamBody, std::io::Error>> + Send>>>; - -/// The server response body type: either a stream (Forward) or fixed bytes/empty (Response). -pub type ServerResponseBody = Either, Empty>>; - -/// Convert a `ProxyResult` to a hyper `Response`. -/// -/// Only handles `Bytes` and `Empty` bodies (LIST, errors, multipart responses). -/// Streaming Forward responses are built directly in `server.rs`. -pub fn build_hyper_response( - result: ProxyResult, -) -> Result, Box> { - let mut builder = Response::builder().status(result.status); - - for (key, value) in result.headers.iter() { - builder = builder.header(key, value); - } - - let body = match result.body { - ProxyResponseBody::Bytes(b) => Either::Right(Either::Left(Full::new(b))), - ProxyResponseBody::Empty => Either::Right(Either::Right(Empty::new())), - }; - - Ok(builder.body(body)?) -} diff --git a/crates/runtimes/server/src/lib.rs b/crates/runtimes/server/src/lib.rs index 43930c8..3fcd749 100644 --- a/crates/runtimes/server/src/lib.rs +++ b/crates/runtimes/server/src/lib.rs @@ -1,12 +1,10 @@ -//! Tokio/Hyper runtime for the S3 proxy gateway. +//! Tokio/axum runtime for the S3 proxy gateway. //! //! This crate provides concrete implementations of the core traits for a -//! standard server environment using Tokio and Hyper. +//! standard server environment using Tokio and axum. //! //! - [`client::ServerBackend`] — implements `ProxyBackend` using reqwest + object_store -//! - [`body`] — converts `ProxyResponseBody` to streaming hyper responses -//! - [`server::run`] — starts the Hyper HTTP server +//! - [`server::run`] — starts the axum HTTP server -pub mod body; pub mod client; pub mod server; diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index cbb3787..4be82fb 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -1,22 +1,21 @@ -//! HTTP server using Hyper, wiring everything together. +//! HTTP server using axum, wiring everything together. -use crate::body::{build_hyper_response, ServerResponseBody}; use crate::client::ServerBackend; -use bytes::Bytes; -use futures::{Stream, TryStreamExt}; -use http::{HeaderMap, Response}; -use http_body_util::{BodyExt, BodyStream, Either, Full, StreamBody}; -use hyper::body::{Frame, Incoming}; -use hyper::service::service_fn; -use hyper_util::rt::{TokioExecutor, TokioIo}; +use axum::body::Body; +use axum::extract::State; +use axum::response::Response; +use axum::Router; +use futures::TryStreamExt; +use http::HeaderMap; +use http_body_util::BodyStream; +use s3_proxy_core::axum::{build_proxy_response, error_response}; use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::proxy::{ - ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, -}; +use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; use s3_proxy_core::resolver::DefaultResolver; +use s3_proxy_sts::{try_handle_sts, JwksCache}; use std::net::SocketAddr; -use std::pin::Pin; use std::sync::Arc; +use std::time::Duration; use tokio::net::TcpListener; /// Server configuration. @@ -36,6 +35,13 @@ impl Default for ServerConfig { } } +struct AppState { + handler: ProxyHandler>, + reqwest_client: reqwest::Client, + sts_config: P, + jwks_cache: JwksCache, +} + /// Run the S3 proxy server. /// /// # Example @@ -47,15 +53,17 @@ impl Default for ServerConfig { /// #[tokio::main] /// async fn main() { /// let config = StaticProvider::from_file("config.toml").unwrap(); +/// let sts_config = config.clone(); /// let server_config = ServerConfig { /// listen_addr: ([0, 0, 0, 0], 8080).into(), /// virtual_host_domain: Some("s3.local".to_string()), /// }; -/// run(config, server_config).await.unwrap(); +/// run(config, sts_config, server_config).await.unwrap(); /// } /// ``` pub async fn run

( config: P, + sts_config: P, server_config: ServerConfig, ) -> Result<(), Box> where @@ -63,80 +71,76 @@ where { let backend = ServerBackend::new(); let reqwest_client = backend.client().clone(); + let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); let resolver = DefaultResolver::new(config, server_config.virtual_host_domain); - let handler = Arc::new(ProxyHandler::new(backend, resolver)); + let handler = ProxyHandler::new(backend, resolver); + + let state = Arc::new(AppState { + handler, + reqwest_client, + sts_config, + jwks_cache, + }); + + let app = Router::new() + .fallback(request_handler::

) + .with_state(state); let listener = TcpListener::bind(server_config.listen_addr).await?; tracing::info!("listening on {}", server_config.listen_addr); - loop { - let (stream, remote_addr) = listener.accept().await?; - let handler = handler.clone(); - let client = reqwest_client.clone(); - - tokio::spawn(async move { - let service = service_fn(move |req: http::Request| { - let handler = handler.clone(); - let client = client.clone(); - - async move { - tracing::debug!( - remote_addr = %remote_addr, - method = %req.method(), - uri = %req.uri(), - "incoming connection" - ); - let result = handle_hyper_request(req, &handler, &client).await; - match result { - Ok(resp) => Ok::<_, hyper::Error>(resp), - Err(e) => { - tracing::error!(remote_addr = %remote_addr, error = %e, "handler error"); - let body = Full::new(Bytes::from(format!("Internal error: {}", e))); - Ok(Response::builder() - .status(500) - .body(Either::Right(Either::Left(body))) - .unwrap()) - } - } - } - }); - - if let Err(err) = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new()) - .serve_connection(TokioIo::new(stream), service) - .await - { - tracing::error!(remote_addr = %remote_addr, error = %err, "connection error"); - } - }); - } + axum::serve(listener, app).await?; + Ok(()) } -async fn handle_hyper_request( - req: http::Request, - handler: &ProxyHandler, - client: &reqwest::Client, -) -> Result, Box> -where - R: s3_proxy_core::resolver::RequestResolver + Send + Sync, -{ - let (parts, incoming_body) = req.into_parts(); +async fn request_handler( + State(state): State>>, + req: axum::extract::Request, +) -> Response { + let (parts, body) = req.into_parts(); let method = parts.method; let uri = parts.uri; let path = uri.path().to_string(); let query = uri.query().map(|q| q.to_string()); let headers = parts.headers; - let action = handler + tracing::debug!( + method = %method, + uri = %uri, + "incoming request" + ); + + // Intercept STS AssumeRoleWithWebIdentity requests + if let Some((status, xml)) = + try_handle_sts(query.as_deref(), &state.sts_config, &state.jwks_cache).await + { + return Response::builder() + .status(status) + .header("content-type", "application/xml") + .body(Body::from(xml)) + .unwrap(); + } + + let action = state + .handler .resolve_request(method, &path, query.as_deref(), &headers) .await; match action { - HandlerAction::Response(result) => build_hyper_response(result), - HandlerAction::Forward(fwd) => forward_to_backend(client, fwd, incoming_body).await, + HandlerAction::Response(result) => build_proxy_response(result), + HandlerAction::Forward(fwd) => { + forward_to_backend(&state.reqwest_client, fwd, body).await + } HandlerAction::NeedsBody(pending) => { - let body = incoming_body.collect().await?.to_bytes(); - let result = handler.handle_with_body(pending, body).await; - build_hyper_response(result) + let collected = match axum::body::to_bytes(body, usize::MAX).await { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "failed to read request body"); + return error_response(500, "Internal error"); + } + }; + let result = state.handler.handle_with_body(pending, collected).await; + build_proxy_response(result) } } } @@ -145,8 +149,8 @@ where async fn forward_to_backend( client: &reqwest::Client, fwd: ForwardRequest, - incoming_body: Incoming, -) -> Result, Box> { + body: Body, +) -> Response { let mut req_builder = client.request(fwd.method.clone(), fwd.url.as_str()); for (k, v) in fwd.headers.iter() { @@ -155,15 +159,18 @@ async fn forward_to_backend( // Attach streaming body for PUT if fwd.method == http::Method::PUT { - let body_stream = BodyStream::new(incoming_body) + let body_stream = BodyStream::new(body) .try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); req_builder = req_builder.body(reqwest::Body::wrap_stream(body_stream)); } - let backend_resp = req_builder.send().await.map_err(|e| { - tracing::error!(error = %e, "forward request failed"); - Box::new(e) as Box - })?; + let backend_resp = match req_builder.send().await { + Ok(resp) => resp, + Err(e) => { + tracing::error!(error = %e, "forward request failed"); + return error_response(502, "Bad Gateway"); + } + }; let status = backend_resp.status().as_u16(); @@ -176,17 +183,12 @@ async fn forward_to_backend( } // Stream the response body - let body_stream = backend_resp.bytes_stream(); - let framed = body_stream - .map_ok(Frame::data) - .map_err(|e| std::io::Error::other(e.to_string())); - let body: ServerResponseBody = Either::Left(StreamBody::new(Box::pin(framed) - as Pin, std::io::Error>> + Send>>)); + let body = Body::from_stream(backend_resp.bytes_stream()); let mut builder = Response::builder().status(status); for (k, v) in resp_headers.iter() { builder = builder.header(k, v); } - Ok(builder.body(body)?) + builder.body(body).unwrap() } From 3d8b1ec2e5094ec77ba9af20ecdf2437431a36eb Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 22:06:07 -0500 Subject: [PATCH 37/82] feat: add cli --- Cargo.lock | 301 ++++++++++++++++++++++++++++++++------- Cargo.toml | 6 + crates/cli/Cargo.toml | 28 ++++ crates/cli/README.md | 76 ++++++++++ crates/cli/src/main.rs | 113 +++++++++++++++ crates/cli/src/oidc.rs | 233 ++++++++++++++++++++++++++++++ crates/cli/src/output.rs | 20 +++ crates/cli/src/sts.rs | 105 ++++++++++++++ 8 files changed, 833 insertions(+), 49 deletions(-) create mode 100644 crates/cli/Cargo.toml create mode 100644 crates/cli/README.md create mode 100644 crates/cli/src/main.rs create mode 100644 crates/cli/src/oidc.rs create mode 100644 crates/cli/src/output.rs create mode 100644 crates/cli/src/sts.rs diff --git a/Cargo.lock b/Cargo.lock index 8c0a19b..40cf7f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.101" @@ -331,6 +381,54 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.22.1" @@ -437,6 +535,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + [[package]] name = "cmake" version = "0.1.57" @@ -446,6 +584,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -471,16 +615,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -1122,12 +1256,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.2", - "system-configuration", "tokio", - "tower-layer", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -1290,6 +1421,31 @@ dependencies = [ "serde", ] +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1415,6 +1571,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -1431,6 +1593,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.1.1" @@ -1546,6 +1714,23 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "open" +version = "5.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb73a7fa3799b198970490a51174027ba0d4ec504b03cd08caf513d40024bc" +dependencies = [ + "is-wsl", + "libc", + "pathdiff", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -1587,6 +1772,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -2083,6 +2274,7 @@ name = "s3-proxy-cf-workers" version = "0.1.0" dependencies = [ "async-trait", + "axum", "bytes", "chrono", "console_error_panic_hook", @@ -2095,6 +2287,7 @@ dependencies = [ "js-sys", "object_store", "quick-xml 0.37.5", + "reqwest", "s3-proxy-core", "s3-proxy-source-coop", "s3-proxy-sts", @@ -2115,6 +2308,7 @@ version = "0.1.0" dependencies = [ "async-trait", "aws-sdk-dynamodb", + "axum", "base64", "bytes", "chrono", @@ -2160,12 +2354,11 @@ dependencies = [ name = "s3-proxy-server" version = "0.1.0" dependencies = [ + "axum", "bytes", "futures", "http 1.4.0", "http-body-util", - "hyper 1.8.1", - "hyper-util", "object_store", "reqwest", "s3-proxy-core", @@ -2175,6 +2368,7 @@ dependencies = [ "thiserror", "tokio", "toml", + "tower-service", "tracing", "tracing-subscriber", ] @@ -2244,7 +2438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ "bitflags", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -2320,6 +2514,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -2433,6 +2638,23 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "source-coop-cli" +version = "0.1.0" +dependencies = [ + "base64", + "clap", + "open", + "quick-xml 0.37.5", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "sha2", + "tokio", + "url", +] + [[package]] name = "spin" version = "0.9.8" @@ -2663,6 +2885,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2700,27 +2928,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "thiserror" version = "2.0.18" @@ -3094,6 +3301,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.21.0" @@ -3363,17 +3576,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -3718,6 +3920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "244647fd7673893058f91f22a0eabd0f45bb50298e995688cb0c4b9837081b19" dependencies = [ "async-trait", + "axum", "bytes", "chrono", "futures-channel", @@ -3725,7 +3928,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "js-sys", - "matchit", + "matchit 0.7.3", "pin-project", "serde", "serde-wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index d546e0e..46d5ae7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/libs/source-coop", "crates/runtimes/server", "crates/runtimes/cf-workers", + "crates/cli", ] # Worker crate is excluded from default builds because it contains !Send # WASM types that only compile correctly for wasm32 targets. Build it @@ -16,6 +17,7 @@ default-members = [ "crates/libs/oidc-provider", "crates/libs/source-coop", "crates/runtimes/server", + "crates/cli", ] resolver = "2" @@ -61,6 +63,10 @@ sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chron # HTTP body http-body = "1" +# Web framework +axum = { version = "0.8", default-features = false } +tower-service = "0.3" + # Server runtime tokio = { version = "1", features = ["full"] } hyper = { version = "1", features = ["full"] } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml new file mode 100644 index 0000000..0f1b724 --- /dev/null +++ b/crates/cli/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "source-coop-cli" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "CLI for Source Cooperative — OIDC login and STS credential exchange" + +[[bin]] +name = "source-coop" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive", "env"] } +open = "5" +reqwest = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +url = { workspace = true } +sha2 = { workspace = true } +base64 = { workspace = true } +rand = "0.8" +quick-xml = { workspace = true } + +[features] +default = ["production"] +production = [] +staging = [] diff --git a/crates/cli/README.md b/crates/cli/README.md new file mode 100644 index 0000000..0508527 --- /dev/null +++ b/crates/cli/README.md @@ -0,0 +1,76 @@ +# source-coop CLI + +Authenticate with the Source Cooperative data proxy and obtain temporary S3 credentials. + +Uses the OAuth2 Authorization Code flow with PKCE to authenticate via browser, then exchanges the OIDC ID token at the proxy's STS endpoint for temporary AWS credentials. + +## Install + +```bash +cargo install --path crates/cli +``` + +## Usage + +```bash +source-coop login --role-arn +``` + +This opens your browser to the Source Cooperative login page. After authenticating, temporary S3 credentials are printed to stdout. + +### Options + +| Flag | Env var | Default | Description | +|------|---------|---------|-------------| +| `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | +| `--client-id` | `SOURCE_OIDC_CLIENT_ID` | `d037d00b-...` | OAuth2 client ID | +| `--proxy-url` | `SOURCE_PROXY_URL` | `http://localhost:8787` | S3 proxy URL for STS | +| `--role-arn` | `SOURCE_ROLE_ARN` | *(required)* | Role ARN to assume | +| `--format` | | `credential-process` | Output format: `credential-process` or `env` | +| `--duration` | | | Session duration in seconds | +| `--scope` | | `openid` | OAuth2 scopes | +| `--port` | | `0` (random) | Local callback port | + +### Output formats + +**credential-process** (default) — for use with `~/.aws/config`: + +```ini +[profile source-coop] +credential_process = source-coop login --role-arn +``` + +**env** — for shell eval: + +```bash +eval $(source-coop login --role-arn --format env) +``` + +## OIDC provider setup + +The CLI uses the OAuth2 Authorization Code flow with PKCE. It starts a temporary local server on `http://127.0.0.1:{port}/callback` to receive the authorization code redirect. + +The OAuth2 client must have a matching redirect URI registered. There are two approaches: + +### Option A: Allow any port (recommended) + +Register `http://127.0.0.1/callback` as a redirect URI on the OAuth2 client. Per [RFC 8252 Section 7.3](https://datatracker.ietf.org/doc/html/rfc8252#section-7.3), loopback redirect URIs should allow any port. Ory Network follows this convention — registering the base URI without a port permits any port. + +The CLI defaults to `--port 0` (OS-assigned random available port), which works with this setup. + +### Option B: Fixed port + +Register a specific redirect URI (e.g. `http://127.0.0.1:8400/callback`) and run the CLI with the matching port: + +```bash +source-coop login --role-arn --port 8400 +``` + +### Client configuration + +The OAuth2 client should be configured as a **public client** (no client secret) with: + +- **Grant type**: Authorization Code +- **Token endpoint auth method**: `none` (public client, PKCE used instead) +- **Allowed scopes**: `openid` +- **Redirect URIs**: `http://127.0.0.1/callback` (see above) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 0000000..4e28679 --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,113 @@ +mod oidc; +mod output; +mod sts; + +use clap::{Parser, Subcommand, ValueEnum}; + +#[cfg(feature = "staging")] +mod defaults { + pub const ISSUER: &str = "https://auth.staging.source.coop"; + pub const CLIENT_ID: &str = "c445cc61-9884-44a8-b051-8d8f7273ffc1"; + pub const PROXY_URL: &str = "https://staging.data.source.coop"; + pub const ROLE_ARN: &str = "source-coop-user"; +} + +#[cfg(not(feature = "staging"))] +mod defaults { + pub const ISSUER: &str = "https://auth.source.coop"; + pub const CLIENT_ID: &str = "d037d00b-09c7-4815-ac39-2a0b9fae40c6"; + pub const PROXY_URL: &str = "https://data.source.coop"; + pub const ROLE_ARN: &str = "source-coop-user"; +} + +#[derive(Parser)] +#[command(name = "source-coop", about = "Source Cooperative CLI")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Authenticate via OIDC and obtain temporary S3 credentials + Login(LoginArgs), +} + +#[derive(Parser)] +struct LoginArgs { + /// OIDC issuer URL + #[arg(long, env = "SOURCE_OIDC_ISSUER", default_value = defaults::ISSUER)] + issuer: String, + + /// OAuth2 client ID + #[arg(long, env = "SOURCE_OIDC_CLIENT_ID", default_value = defaults::CLIENT_ID)] + client_id: String, + + /// S3 proxy URL for STS + #[arg(long, env = "SOURCE_PROXY_URL", default_value = defaults::PROXY_URL)] + proxy_url: String, + + /// Role ARN to assume + #[arg(long, env = "SOURCE_ROLE_ARN", default_value = defaults::ROLE_ARN)] + role_arn: String, + + /// Output format + #[arg(long, default_value = "credential-process")] + format: OutputFormat, + + /// Session duration in seconds + #[arg(long)] + duration: Option, + + /// OAuth2 scopes + #[arg(long, default_value = "openid")] + scope: String, + + /// Local callback port (0 for random available port) + #[arg(long, default_value = "0")] + port: u16, +} + +#[derive(Clone, ValueEnum)] +enum OutputFormat { + /// AWS credential_process JSON format + CredentialProcess, + /// Shell export statements + Env, +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + + match cli.command { + Commands::Login(args) => { + if let Err(e) = run_login(args).await { + eprintln!("Error: {e}"); + std::process::exit(1); + } + } + } +} + +async fn run_login(args: LoginArgs) -> Result<(), String> { + // 1. OIDC Discovery + eprintln!("Discovering OIDC endpoints..."); + let endpoints = oidc::discover(&args.issuer).await?; + + // 2. Browser-based OIDC login + let id_token = oidc::login(&endpoints, &args.client_id, &args.scope, args.port).await?; + eprintln!("Authentication successful."); + + // 3. STS credential exchange + eprintln!("Exchanging token for credentials..."); + let creds = sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; + + // 4. Output + match args.format { + OutputFormat::CredentialProcess => output::print_credential_process(&creds), + OutputFormat::Env => output::print_env(&creds), + } + + Ok(()) +} diff --git a/crates/cli/src/oidc.rs b/crates/cli/src/oidc.rs new file mode 100644 index 0000000..5ff815a --- /dev/null +++ b/crates/cli/src/oidc.rs @@ -0,0 +1,233 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use rand::Rng; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use tokio::net::TcpListener; +use url::Url; + +#[derive(Debug)] +pub struct OidcEndpoints { + pub authorization_endpoint: String, + pub token_endpoint: String, +} + +/// Fetch OIDC discovery document and extract endpoints. +pub async fn discover(issuer: &str) -> Result { + let discovery_url = format!( + "{}/.well-known/openid-configuration", + issuer.trim_end_matches('/') + ); + + let resp = reqwest::get(&discovery_url) + .await + .map_err(|e| format!("Failed to fetch OIDC discovery document: {e}"))?; + + if !resp.status().is_success() { + return Err(format!( + "OIDC discovery returned status {}", + resp.status() + )); + } + + let doc: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("Failed to parse OIDC discovery document: {e}"))?; + + let authorization_endpoint = doc["authorization_endpoint"] + .as_str() + .ok_or("Missing authorization_endpoint in discovery document")? + .to_string(); + + let token_endpoint = doc["token_endpoint"] + .as_str() + .ok_or("Missing token_endpoint in discovery document")? + .to_string(); + + Ok(OidcEndpoints { + authorization_endpoint, + token_endpoint, + }) +} + +/// Run the browser-based OAuth2 Authorization Code flow with PKCE. +/// Opens the user's browser to the OIDC provider, waits for the callback, +/// and returns the `id_token`. +pub async fn login( + endpoints: &OidcEndpoints, + client_id: &str, + scope: &str, + port: u16, +) -> Result { + let pkce = generate_pkce(); + let state: String = URL_SAFE_NO_PAD.encode(rand::thread_rng().gen::<[u8; 16]>()); + + // Bind local callback server + let listener = TcpListener::bind(format!("127.0.0.1:{port}")) + .await + .map_err(|e| format!("Failed to bind local server: {e}"))?; + + let local_addr = listener + .local_addr() + .map_err(|e| format!("Failed to get local address: {e}"))?; + let redirect_uri = format!("http://127.0.0.1:{}/callback", local_addr.port()); + + // Build authorization URL + let mut auth_url = Url::parse(&endpoints.authorization_endpoint) + .map_err(|e| format!("Invalid authorization endpoint URL: {e}"))?; + auth_url + .query_pairs_mut() + .append_pair("response_type", "code") + .append_pair("client_id", client_id) + .append_pair("redirect_uri", &redirect_uri) + .append_pair("scope", scope) + .append_pair("code_challenge", &pkce.challenge) + .append_pair("code_challenge_method", "S256") + .append_pair("state", &state); + + eprintln!("Opening browser for authentication..."); + if open::that(auth_url.as_str()).is_err() { + eprintln!( + "Could not open browser automatically. Please open this URL:\n{}", + auth_url + ); + } + + // Wait for callback + let (code, received_state) = wait_for_callback(&listener).await?; + + if received_state != state { + return Err("State mismatch — possible CSRF attack".to_string()); + } + + // Exchange code for tokens + exchange_code( + &endpoints.token_endpoint, + &code, + &redirect_uri, + client_id, + &pkce.verifier, + ) + .await +} + +/// Accept a single HTTP request on the callback listener, extract `code` and `state`. +async fn wait_for_callback(listener: &TcpListener) -> Result<(String, String), String> { + let (stream, _) = listener + .accept() + .await + .map_err(|e| format!("Failed to accept callback connection: {e}"))?; + + let std_stream = stream + .into_std() + .map_err(|e| format!("Failed to convert stream: {e}"))?; + std_stream + .set_nonblocking(false) + .map_err(|e| format!("Failed to set blocking: {e}"))?; + + let mut reader = BufReader::new(&std_stream); + let mut request_line = String::new(); + reader + .read_line(&mut request_line) + .map_err(|e| format!("Failed to read request: {e}"))?; + + let path = request_line + .split_whitespace() + .nth(1) + .ok_or("Invalid HTTP request")?; + + let url = Url::parse(&format!("http://localhost{path}")) + .map_err(|e| format!("Failed to parse callback URL: {e}"))?; + + let params: HashMap = url.query_pairs().into_owned().collect(); + + if let Some(error) = params.get("error") { + let desc = params + .get("error_description") + .map(|d| format!(": {d}")) + .unwrap_or_default(); + let html = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n\ +

Authentication Failed

{error}{desc}

\ +

You can close this tab.

" + ); + let _ = (&std_stream).write_all(html.as_bytes()); + return Err(format!("Authentication error: {error}{desc}")); + } + + let code = params + .get("code") + .ok_or("No authorization code in callback")? + .clone(); + let received_state = params.get("state").ok_or("No state in callback")?.clone(); + + let html = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n\ +

Authentication Successful

\ +

You can close this tab and return to your terminal.

"; + (&std_stream) + .write_all(html.as_bytes()) + .map_err(|e| format!("Failed to send response: {e}"))?; + + Ok((code, received_state)) +} + +/// Exchange authorization code for tokens, return the `id_token`. +async fn exchange_code( + token_endpoint: &str, + code: &str, + redirect_uri: &str, + client_id: &str, + code_verifier: &str, +) -> Result { + let client = reqwest::Client::new(); + let resp = client + .post(token_endpoint) + .form(&[ + ("grant_type", "authorization_code"), + ("code", code), + ("redirect_uri", redirect_uri), + ("client_id", client_id), + ("code_verifier", code_verifier), + ]) + .send() + .await + .map_err(|e| format!("Token exchange request failed: {e}"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("Token exchange failed (HTTP {status}): {body}")); + } + + let body: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("Failed to parse token response: {e}"))?; + + body["id_token"] + .as_str() + .map(|s| s.to_string()) + .ok_or_else(|| "No id_token in token response".to_string()) +} + +struct Pkce { + verifier: String, + challenge: String, +} + +fn generate_pkce() -> Pkce { + let mut rng = rand::thread_rng(); + let bytes: Vec = (0..32).map(|_| rng.gen()).collect(); + let verifier = URL_SAFE_NO_PAD.encode(&bytes); + + let mut hasher = Sha256::new(); + hasher.update(verifier.as_bytes()); + let challenge = URL_SAFE_NO_PAD.encode(hasher.finalize()); + + Pkce { + verifier, + challenge, + } +} diff --git a/crates/cli/src/output.rs b/crates/cli/src/output.rs new file mode 100644 index 0000000..1307e8a --- /dev/null +++ b/crates/cli/src/output.rs @@ -0,0 +1,20 @@ +use crate::sts::Credentials; + +/// Print credentials in AWS credential_process JSON format. +pub fn print_credential_process(creds: &Credentials) { + let json = serde_json::json!({ + "Version": 1, + "AccessKeyId": creds.access_key_id, + "SecretAccessKey": creds.secret_access_key, + "SessionToken": creds.session_token, + "Expiration": creds.expiration, + }); + println!("{}", serde_json::to_string_pretty(&json).unwrap()); +} + +/// Print credentials as shell export statements. +pub fn print_env(creds: &Credentials) { + println!("export AWS_ACCESS_KEY_ID={}", creds.access_key_id); + println!("export AWS_SECRET_ACCESS_KEY={}", creds.secret_access_key); + println!("export AWS_SESSION_TOKEN={}", creds.session_token); +} diff --git a/crates/cli/src/sts.rs b/crates/cli/src/sts.rs new file mode 100644 index 0000000..313ef37 --- /dev/null +++ b/crates/cli/src/sts.rs @@ -0,0 +1,105 @@ +use quick_xml::de::from_str as xml_from_str; +use serde::Deserialize; + +#[derive(Debug, Clone)] +pub struct Credentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: String, +} + +/// Call the proxy's STS AssumeRoleWithWebIdentity endpoint. +pub async fn assume_role( + proxy_url: &str, + role_arn: &str, + web_identity_token: &str, + duration_seconds: Option, +) -> Result { + let mut url = url::Url::parse(proxy_url) + .map_err(|e| format!("Invalid proxy URL: {e}"))?; + + url.query_pairs_mut() + .append_pair("Action", "AssumeRoleWithWebIdentity") + .append_pair("RoleArn", role_arn) + .append_pair("WebIdentityToken", web_identity_token); + + if let Some(duration) = duration_seconds { + url.query_pairs_mut() + .append_pair("DurationSeconds", &duration.to_string()); + } + + let resp = reqwest::get(url.as_str()) + .await + .map_err(|e| format!("STS request failed: {e}"))?; + + let status = resp.status(); + let body = resp + .text() + .await + .map_err(|e| format!("Failed to read STS response: {e}"))?; + + if !status.is_success() { + // Try to parse error XML for a better message + if let Ok(err) = xml_from_str::(&body) { + return Err(format!( + "STS error ({}): {}", + err.error.code, err.error.message + )); + } + return Err(format!("STS request failed (HTTP {status}): {body}")); + } + + let parsed: StsResponse = + xml_from_str(&body).map_err(|e| format!("Failed to parse STS response XML: {e}"))?; + + let creds = parsed.result.credentials; + Ok(Credentials { + access_key_id: creds.access_key_id, + secret_access_key: creds.secret_access_key, + session_token: creds.session_token, + expiration: creds.expiration, + }) +} + +// XML deserialization types matching the STS response format + +#[derive(Debug, Deserialize)] +#[serde(rename = "AssumeRoleWithWebIdentityResponse")] +struct StsResponse { + #[serde(rename = "AssumeRoleWithWebIdentityResult")] + result: StsResult, +} + +#[derive(Debug, Deserialize)] +struct StsResult { + #[serde(rename = "Credentials")] + credentials: StsCredentials, +} + +#[derive(Debug, Deserialize)] +struct StsCredentials { + #[serde(rename = "AccessKeyId")] + access_key_id: String, + #[serde(rename = "SecretAccessKey")] + secret_access_key: String, + #[serde(rename = "SessionToken")] + session_token: String, + #[serde(rename = "Expiration")] + expiration: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename = "ErrorResponse")] +struct StsErrorResponse { + #[serde(rename = "Error")] + error: StsError, +} + +#[derive(Debug, Deserialize)] +struct StsError { + #[serde(rename = "Code")] + code: String, + #[serde(rename = "Message")] + message: String, +} From 9b80406365f0cf3bb760e003ed587df3797432d7 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 22:42:39 -0500 Subject: [PATCH 38/82] feat: support substitutions in role policies --- config.example.toml | 15 ++++ crates/libs/sts/src/lib.rs | 2 +- crates/libs/sts/src/sts.rs | 110 ++++++++++++++++++++++- crates/runtimes/cf-workers/wrangler.toml | 18 +++- 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/config.example.toml b/config.example.toml index 4ffabcb..b35253d 100644 --- a/config.example.toml +++ b/config.example.toml @@ -89,6 +89,21 @@ bucket = "deploy-bundles" prefixes = [] # full bucket access actions = ["get_object", "head_object", "put_object", "create_multipart_upload", "upload_part", "complete_multipart_upload"] +# Role for Source Cooperative CLI users (OIDC login via Ory) +[[roles]] +role_id = "source-user" +name = "Source Cooperative User" +trusted_oidc_issuers = ["https://auth.source.coop", "https://auth.staging.source.coop"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +# Template variables like {sub} are resolved against the user's JWT claims. +# This gives each user read/write access to a bucket matching their username. +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] + # Read-only role for CI [[roles]] role_id = "ci-readonly" diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index 6ea54b7..091d623 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -149,7 +149,7 @@ pub async fn assume_role_with_web_identity( .unwrap_or(3600) .clamp(MIN_SESSION_DURATION_SECS, role.max_session_duration_secs); - let creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix); + let creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix, &claims); // Store them config.store_temporary_credential(&creds).await?; diff --git a/crates/libs/sts/src/sts.rs b/crates/libs/sts/src/sts.rs index 70f975b..19e846d 100644 --- a/crates/libs/sts/src/sts.rs +++ b/crates/libs/sts/src/sts.rs @@ -1,15 +1,63 @@ //! STS credential minting. use chrono::{Duration, Utc}; -use s3_proxy_core::types::{RoleConfig, TemporaryCredentials}; +use s3_proxy_core::types::{AccessScope, RoleConfig, TemporaryCredentials}; use uuid::Uuid; +/// Resolve `{claim_name}` template variables in access scopes against JWT claims. +/// +/// Each `{name}` in `bucket` or `prefixes` is replaced with the corresponding +/// string claim value. Missing or non-string claims resolve to an empty string, +/// which will safely fail authorization downstream. +fn resolve_scopes(scopes: &[AccessScope], claims: &serde_json::Value) -> Vec { + scopes + .iter() + .map(|scope| { + let bucket = resolve_template(&scope.bucket, claims); + let prefixes = scope + .prefixes + .iter() + .map(|p| resolve_template(p, claims)) + .collect(); + AccessScope { + bucket, + prefixes, + actions: scope.actions.clone(), + } + }) + .collect() +} + +/// Replace all `{key}` placeholders in `template` with values from `claims`. +fn resolve_template(template: &str, claims: &serde_json::Value) -> String { + let mut result = template.to_string(); + // Find all {…} placeholders and replace them + while let Some(start) = result.find('{') { + if let Some(end) = result[start..].find('}') { + let end = start + end; + let key = &result[start + 1..end]; + let value = claims + .get(key) + .and_then(|v| v.as_str()) + .unwrap_or(""); + result = format!("{}{}{}", &result[..start], value, &result[end + 1..]); + } else { + break; + } + } + result +} + /// Mint a new set of temporary credentials for an assumed role. +/// +/// Template variables (`{claim_name}`) in `role.allowed_scopes` are resolved +/// against the provided JWT `claims` before being stored in the credentials. pub fn mint_temporary_credentials( role: &RoleConfig, source_identity: &str, duration_seconds: u64, key_prefix: &str, + claims: &serde_json::Value, ) -> TemporaryCredentials { let access_key_id = format!("{}{}", key_prefix, generate_random_id(16)); let secret_access_key = generate_random_id(40); @@ -22,7 +70,7 @@ pub fn mint_temporary_credentials( secret_access_key, session_token, expiration, - allowed_scopes: role.allowed_scopes.clone(), + allowed_scopes: resolve_scopes(&role.allowed_scopes, claims), assumed_role_id: role.role_id.clone(), source_identity: source_identity.to_string(), } @@ -51,3 +99,61 @@ fn rand_byte() -> u8 { let id = Uuid::new_v4(); id.as_bytes()[0] } + +#[cfg(test)] +mod tests { + use super::*; + use s3_proxy_core::types::Action; + use serde_json::json; + + fn scope(bucket: &str, prefixes: &[&str], actions: &[Action]) -> AccessScope { + AccessScope { + bucket: bucket.to_string(), + prefixes: prefixes.iter().map(|s| s.to_string()).collect(), + actions: actions.to_vec(), + } + } + + #[test] + fn resolve_template_in_bucket() { + let scopes = vec![scope("{sub}", &[], &[Action::GetObject])]; + let claims = json!({"sub": "alice"}); + let resolved = resolve_scopes(&scopes, &claims); + assert_eq!(resolved[0].bucket, "alice"); + } + + #[test] + fn resolve_template_in_prefix() { + let scopes = vec![scope("my-bucket", &["data/{sub}/"], &[Action::GetObject])]; + let claims = json!({"sub": "alice"}); + let resolved = resolve_scopes(&scopes, &claims); + assert_eq!(resolved[0].prefixes[0], "data/alice/"); + } + + #[test] + fn resolve_multiple_claims() { + let scopes = vec![scope("{org}", &["{sub}/"], &[Action::GetObject])]; + let claims = json!({"sub": "alice", "org": "acme"}); + let resolved = resolve_scopes(&scopes, &claims); + assert_eq!(resolved[0].bucket, "acme"); + assert_eq!(resolved[0].prefixes[0], "alice/"); + } + + #[test] + fn no_templates_unchanged() { + let scopes = vec![scope("static-bucket", &["prefix/"], &[Action::GetObject])]; + let claims = json!({"sub": "alice"}); + let resolved = resolve_scopes(&scopes, &claims); + assert_eq!(resolved[0].bucket, "static-bucket"); + assert_eq!(resolved[0].prefixes[0], "prefix/"); + } + + #[test] + fn missing_claim_resolves_to_empty() { + let scopes = vec![scope("{missing}", &["{also_missing}/"], &[Action::GetObject])]; + let claims = json!({"sub": "alice"}); + let resolved = resolve_scopes(&scopes, &claims); + assert_eq!(resolved[0].bucket, ""); + assert_eq!(resolved[0].prefixes[0], "/"); + } +} diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index 737479f..63920b2 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -11,7 +11,6 @@ VIRTUAL_HOST_DOMAIN = "s3.local" # For production, consider storing this in Workers KV or a Secrets binding. [vars.PROXY_CONFIG] -roles = [] [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] @@ -89,3 +88,20 @@ actions = [ ] bucket = "private-uploads" prefixes = [] + +[vars.STS_CONFIG] +[[vars.STS_CONFIG.roles]] +max_session_duration_secs = 3600 +name = "Source Cooperative User" +role_id = "source-coop-user" +subject_conditions = ["*"] +trusted_oidc_issuers = [ + "https://auth.source.coop", + "https://auth.staging.source.coop", + "https://optimistic-jackson-tvx6h5ig8s.projects.oryapis.com", +] + +[[vars.STS_CONFIG.roles.allowed_scopes]] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +bucket = "{sub}" # this user +prefixes = [] From f3bfd87743e74dae6c5077b37ff9329260d98c68 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 22:45:16 -0500 Subject: [PATCH 39/82] chore: add build helper --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index eb1cc63..c5ed2ef 100644 --- a/Makefile +++ b/Makefile @@ -23,3 +23,8 @@ run\:server: run\:workers: npx wrangler dev --cwd crates/runtimes/cf-workers +build\:cli: + cargo build -p source-coop-cli + +build\:cli\:staging: + cargo build -p source-coop-cli --no-default-features --features staging From ed8ad7dc58e12c97d098e8023668d73071e22e7c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 22:48:25 -0500 Subject: [PATCH 40/82] chore: fix cargo.lock --- Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40cf7f3..ce5a537 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2271,7 +2271,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3-proxy-cf-workers" -version = "0.1.0" +version = "1.0.4" dependencies = [ "async-trait", "axum", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "s3-proxy-core" -version = "0.1.0" +version = "1.0.4" dependencies = [ "async-trait", "aws-sdk-dynamodb", @@ -2333,7 +2333,7 @@ dependencies = [ [[package]] name = "s3-proxy-oidc-provider" -version = "0.1.0" +version = "1.0.4" dependencies = [ "async-trait", "base64", @@ -2352,7 +2352,7 @@ dependencies = [ [[package]] name = "s3-proxy-server" -version = "0.1.0" +version = "1.0.4" dependencies = [ "axum", "bytes", @@ -2375,7 +2375,7 @@ dependencies = [ [[package]] name = "s3-proxy-source-coop" -version = "0.1.0" +version = "1.0.4" dependencies = [ "bytes", "http 1.4.0", @@ -2388,7 +2388,7 @@ dependencies = [ [[package]] name = "s3-proxy-sts" -version = "0.1.0" +version = "1.0.4" dependencies = [ "async-trait", "base64", @@ -2640,7 +2640,7 @@ dependencies = [ [[package]] name = "source-coop-cli" -version = "0.1.0" +version = "1.0.4" dependencies = [ "base64", "clap", From 9183685299d47a11ce4a04848c211c5c60397d8a Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 22:49:33 -0500 Subject: [PATCH 41/82] chore: rm unused file --- src/main.rs | 509 ---------------------------------------------------- 1 file changed, 509 deletions(-) delete mode 100644 src/main.rs diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 6abce68..0000000 --- a/src/main.rs +++ /dev/null @@ -1,509 +0,0 @@ -mod apis; -mod backends; -mod utils; -use crate::utils::core::{split_at_first_slash, StreamingResponse}; -use actix_cors::Cors; -use actix_web::body::{BodySize, BoxBody, MessageBody}; -use actix_web::error::ErrorInternalServerError; -use actix_web::{ - delete, get, head, http::header::CONTENT_TYPE, http::header::RANGE, middleware, post, put, web, - App, HttpRequest, HttpResponse, HttpServer, Responder, -}; - -use apis::source::{RepositoryPermission, SourceApi}; -use apis::Api; -use backends::common::{CommonPrefix, CompleteMultipartUpload, ListBucketResult}; -use bytes::Bytes; -use core::num::NonZeroU32; -use env_logger::Env; -use futures_util::StreamExt; -use quick_xml::se::to_string_with_root; -use serde::Deserialize; -use serde_xml_rs::from_str; -use std::env; -use std::fmt::Debug; -use std::pin::Pin; -use std::str::from_utf8; -use std::task::{Context, Poll}; -use utils::auth::{LoadIdentity, UserIdentity}; -use utils::errors::BackendError; -const VERSION: &str = env!("CARGO_PKG_VERSION"); - -struct FakeBody { - size: usize, -} - -impl MessageBody for FakeBody { - type Error = actix_web::Error; - - fn size(&self) -> BodySize { - BodySize::Sized(self.size as u64) - } - - fn poll_next( - self: Pin<&mut Self>, - _: &mut Context<'_>, - ) -> Poll>> { - Poll::Ready(None) - } -} - -#[get("/{account_id}/{repository_id}/{key:.*}")] -async fn get_object( - api_client: web::Data, - req: HttpRequest, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - let mut range_start = 0; - let mut is_range_request = false; - - let range = headers - .get(RANGE) - .and_then(|h| h.to_str().ok()) - .and_then(|r| r.strip_prefix("bytes=")) - .and_then(|bytes_range| bytes_range.split_once('-')) - .and_then(|(start, end)| { - start.parse::().ok().map(|s| { - range_start = s; - if end.is_empty() || end.parse::().is_ok() { - is_range_request = true; - Some(format!("bytes={start}-{end}")) - } else { - None - } - }) - }) - .flatten(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Read, - ) - .await?; - - // Found the repository, now try to get the object - let res = client.get_object(key.clone(), range).await?; - - let mut content_length = String::from("*"); - // Remove this if statement to increase performance since it's making an extra request just to get the total content-length - // This is only needed for range requests and in theory, you can return a * in the Content-Range header to indicate that the content length is unknown - if is_range_request { - content_length = client - .head_object(key.clone()) - .await? - .content_length - .to_string(); - } - - let stream = res - .body - .map(|result| result.map_err(|e| ErrorInternalServerError(e.to_string()))); - - let streaming_response = StreamingResponse::new(stream, res.content_length); - let mut response = if is_range_request { - HttpResponse::PartialContent() - } else { - HttpResponse::Ok() - }; - - let mut response = response - .insert_header(("Accept-Ranges", "bytes")) - .insert_header(("Access-Control-Expose-Headers", "Accept-Ranges")) - .insert_header(("Content-Type", res.content_type)) - .insert_header(("Last-Modified", res.last_modified)) - .insert_header(("Content-Length", res.content_length.to_string())) - .insert_header(("ETag", res.etag)); - - if is_range_request { - response = response - .insert_header(( - "Content-Range", - format!( - "bytes {}-{}/{}", - range_start, - range_start + res.content_length - 1, - content_length - ), - )) - .insert_header(( - "Access-Control-Expose-Headers", - "Accept-Ranges, Content-Range", - )); - } - - Ok(response.body(streaming_response)) -} - -#[derive(Debug, Deserialize)] -struct DeleteParams { - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[delete("/{account_id}/{repository_id}/{key:.*}")] -async fn delete_object( - api_client: web::Data, - params: web::Query, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.upload_id.is_none() { - // Found the repository, now try to delete the object - client.delete_object(key.clone()).await?; - Ok(HttpResponse::NoContent().finish()) - } else { - client - .abort_multipart_upload(key.clone(), params.upload_id.clone().unwrap()) - .await?; - Ok(HttpResponse::NoContent().finish()) - } -} - -#[derive(Debug, Deserialize)] -struct PutParams { - #[serde(rename = "partNumber")] - part_number: Option, - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[put("/{account_id}/{repository_id}/{key:.*}")] -async fn put_object( - api_client: web::Data, - req: HttpRequest, - bytes: Bytes, - params: web::Query, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.part_number.is_none() && params.upload_id.is_none() { - // Check if this is a server-side copy operation - if let Some(header_copy_identifier) = req.headers().get("x-amz-copy-source") { - let copy_identifier_path = header_copy_identifier.to_str().unwrap_or(""); - client - .copy_object((©_identifier_path).to_string(), key.clone(), None) - .await?; - Ok(HttpResponse::NoContent().finish()) - } else { - // Found the repository, now try to upload the object - client - .put_object( - key.clone(), - bytes, - headers - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .map(|s| s.to_string()), - ) - .await?; - Ok(HttpResponse::NoContent().finish()) - } - } else if params.part_number.is_some() && params.upload_id.is_some() { - let res = client - .upload_multipart_part( - key.clone(), - params.upload_id.clone().unwrap(), - params.part_number.clone().unwrap(), - bytes, - ) - .await?; - Ok(HttpResponse::Ok() - .insert_header(("ETag", res.etag)) - .finish()) - } else { - Err(BackendError::InvalidRequest( - "Must provide both part number and upload id or neither.".to_string(), - )) - } -} - -#[derive(Debug, Deserialize)] -struct PostParams { - uploads: Option, - #[serde(rename = "uploadId")] - upload_id: Option, -} - -#[post("/{account_id}/{repository_id}/{key:.*}")] -async fn post_handler( - api_client: web::Data, - req: HttpRequest, - params: web::Query, - mut payload: web::Payload, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - let headers = req.headers(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Write, - ) - .await?; - - if params.uploads.is_some() { - let res = client - .create_multipart_upload( - key, - headers - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .map(|s| s.to_string()), - ) - .await?; - let serialized = to_string_with_root("InitiateMultipartUploadResult", &res)?; - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) - } else if params.upload_id.is_some() { - let mut body = String::new(); - while let Some(chunk) = payload.next().await { - match chunk { - Ok(chunk) => match from_utf8(&chunk) { - Ok(s) => body.push_str(s), - Err(_) => { - return Err(BackendError::InvalidRequest("Invalid UTF-8".to_string())) - } - }, - Err(err) => return Err(BackendError::UnexpectedApiError(err.to_string())), - } - } - - let upload = from_str::(&body)?; - let res = client - .complete_multipart_upload(key, params.upload_id.clone().unwrap(), upload.parts) - .await?; - let serialized = to_string_with_root("CompleteMultipartUploadResult", &res)?; - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) - } else { - Err(BackendError::InvalidRequest( - "Must provide either uploads or uploadId".to_string(), - )) - } -} - -#[head("/{account_id}/{repository_id}/{key:.*}")] -async fn head_object( - api_client: web::Data, - path: web::Path<(String, String, String)>, - user_identity: web::ReqData, -) -> Result { - let (account_id, repository_id, key) = path.into_inner(); - - let client = api_client - .get_backend_client(&account_id, &repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - &repository_id, - RepositoryPermission::Read, - ) - .await?; - - let res = client.head_object(key.clone()).await?; - Ok(HttpResponse::Ok() - .insert_header(("Accept-Ranges", "bytes")) - .insert_header(("Access-Control-Expose-Headers", "Accept-Ranges")) - .insert_header(("Content-Type", res.content_type)) - .insert_header(("Last-Modified", res.last_modified)) - .insert_header(("ETag", res.etag)) - .body(BoxBody::new(FakeBody { - size: res.content_length as usize, - }))) -} - -#[derive(Deserialize)] -struct ListObjectsV2Query { - #[serde(rename = "prefix")] - prefix: Option, - #[serde(rename = "list-type")] - _list_type: u8, - #[serde(rename = "max-keys")] - max_keys: Option, - #[serde(rename = "delimiter")] - delimiter: Option, - #[serde(rename = "continuation-token")] - continuation_token: Option, -} - -#[get("/{account_id}")] -async fn list_objects( - api_client: web::Data, - info: web::Query, - path: web::Path, - user_identity: web::ReqData, -) -> Result { - let account_id = path.into_inner(); - - if info.prefix.clone().is_some_and(|s| s.is_empty()) || info.prefix.is_none() { - let account = api_client - .get_account(account_id.clone(), (*user_identity).clone()) - .await?; - - let repositories = account.repositories; - let mut common_prefixes = Vec::new(); - for repository_id in repositories.iter() { - common_prefixes.push(CommonPrefix { - prefix: format!("{}/", repository_id.clone()), - }); - } - let list_response = ListBucketResult { - name: account_id.clone(), - prefix: "/".to_string(), - key_count: 0, - max_keys: 0, - is_truncated: false, - contents: vec![], - common_prefixes, - next_continuation_token: None, - }; - - let serialized = to_string_with_root("ListBucketResult", &list_response)?; - return Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)); - } - - let path_prefix = info.prefix.clone().unwrap_or("".to_string()); - - let (repository_id, prefix) = split_at_first_slash(&path_prefix); - - let mut max_keys = NonZeroU32::new(1000).unwrap(); - if let Some(mk) = info.max_keys { - max_keys = mk; - } - - let client = api_client - .get_backend_client(&account_id, repository_id) - .await?; - - api_client - .assert_authorized( - user_identity.into_inner(), - &account_id, - repository_id, - RepositoryPermission::Read, - ) - .await?; - - // We're listing within a repository, so we need to query the object store backend - let res = client - .list_objects_v2( - prefix.to_string(), - info.continuation_token.clone(), - info.delimiter.clone(), - max_keys, - ) - .await?; - - let serialized = to_string_with_root("ListBucketResult", &res)?; - - Ok(HttpResponse::Ok() - .content_type("application/xml") - .body(serialized)) -} - -#[get("/")] -async fn index() -> impl Responder { - HttpResponse::Ok().body(format!("Source Cooperative Data Proxy v{VERSION}")) -} - -// Main function to set up and run the HTTP server -#[actix_web::main] -async fn main() -> std::io::Result<()> { - let source_api_url = env::var("SOURCE_API_URL").expect("SOURCE_API_URL must be set"); - let source_api_key = env::var("SOURCE_API_KEY").expect("SOURCE_API_KEY must be set"); - let source_api_proxy_url = env::var("SOURCE_API_PROXY_URL").ok(); // Optional proxy for the Source API - let source_api = web::Data::new(SourceApi::new( - source_api_url, - source_api_key, - source_api_proxy_url, - )); - env_logger::init_from_env(Env::default().default_filter_or("info")); - - HttpServer::new(move || { - App::new() - .app_data(web::PayloadConfig::new(1024 * 1024 * 50)) - .app_data(source_api.clone()) - .app_data(web::Data::new(UserIdentity { api_key: None })) - .wrap( - // Configure CORS - Cors::default() - .allow_any_origin() - .allow_any_method() - .allow_any_header() - .supports_credentials() - .block_on_origin_mismatch(false) - .max_age(3600), - ) - .wrap(middleware::NormalizePath::trim()) - .wrap(middleware::DefaultHeaders::new().add(("X-Version", VERSION))) - .wrap(middleware::Logger::default()) - .wrap(LoadIdentity) - // Register the endpoints - .service(get_object) - .service(delete_object) - .service(post_handler) - .service(put_object) - .service(head_object) - .service(list_objects) - .service(index) - }) - .bind("0.0.0.0:8080")? - .run() - .await -} From 38af07ac04af7d41fbc27ba350653637dc589ed2 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 23:26:07 -0500 Subject: [PATCH 42/82] chore: rename modules --- CLAUDE.md | 6 +- Cargo.lock | 274 +++++++++--------- Cargo.toml | 14 +- Dockerfile | 8 +- Makefile | 4 +- README.md | 118 ++++++-- crates/libs/{source-coop => api}/Cargo.toml | 4 +- crates/libs/{source-coop => api}/src/api.rs | 4 +- crates/libs/{source-coop => api}/src/lib.rs | 0 .../libs/{source-coop => api}/src/resolver.rs | 10 +- crates/libs/core/Cargo.toml | 2 +- crates/libs/core/README.md | 14 +- crates/libs/core/src/config/cached.rs | 2 +- crates/libs/core/src/config/dynamodb.rs | 2 +- crates/libs/core/src/config/http.rs | 2 +- crates/libs/core/src/config/mod.rs | 2 +- crates/libs/core/src/config/postgres.rs | 2 +- crates/libs/core/src/maybe_send.rs | 2 +- crates/libs/oidc-provider/Cargo.toml | 4 +- crates/libs/oidc-provider/src/exchange/mod.rs | 4 +- crates/libs/oidc-provider/src/lib.rs | 10 +- crates/libs/sts/Cargo.toml | 4 +- crates/libs/sts/README.md | 8 +- crates/libs/sts/src/jwks.rs | 4 +- crates/libs/sts/src/lib.rs | 6 +- crates/libs/sts/src/request.rs | 2 +- crates/libs/sts/src/responses.rs | 4 +- crates/libs/sts/src/sts.rs | 4 +- crates/runtimes/cf-workers/Cargo.toml | 8 +- crates/runtimes/cf-workers/README.md | 10 +- crates/runtimes/cf-workers/src/client.rs | 8 +- crates/runtimes/cf-workers/src/lib.rs | 14 +- crates/runtimes/cf-workers/wrangler.toml | 2 +- crates/runtimes/server/Cargo.toml | 8 +- crates/runtimes/server/README.md | 30 +- .../bin/{s3-proxy.rs => source-coop-proxy.rs} | 14 +- crates/runtimes/server/src/client.rs | 6 +- crates/runtimes/server/src/server.rs | 14 +- 38 files changed, 345 insertions(+), 289 deletions(-) rename crates/libs/{source-coop => api}/Cargo.toml (84%) rename crates/libs/{source-coop => api}/src/api.rs (98%) rename crates/libs/{source-coop => api}/src/lib.rs (100%) rename crates/libs/{source-coop => api}/src/resolver.rs (98%) rename crates/runtimes/server/src/bin/{s3-proxy.rs => source-coop-proxy.rs} (80%) diff --git a/CLAUDE.md b/CLAUDE.md index ae96a87..f87b332 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ cargo check cargo build # CF Workers crate MUST be checked/built with the wasm32 target: -cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown # Run tests cargo test @@ -34,7 +34,7 @@ cargo test - **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has three methods: `create_store()` returns an `Arc` for LIST, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: - **Server**: `ServerBackend` delegates to `build_object_store()` (with default connector) and `build_signer()`, and uses reqwest for raw HTTP + Forward execution. - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` (injecting `FetchConnector`) and `build_signer()`, and uses `web_sys::fetch` for raw HTTP + Forward execution. -- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_object_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_object_store()` only — `build_signer()` needs no connector since signing is pure computation. +- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_object_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_object_store()` only — `build_signer()` needs no connector since signing is pure computation. - **Operation dispatch** via presigned URLs and direct object_store: - **GET/HEAD/PUT/DELETE** → `create_signer()` generates a presigned URL, returned as `HandlerAction::Forward`. The runtime executes the URL with its native HTTP client, streaming request/response bodies directly without handler involvement. - **LIST** → `create_store()` + `store.list_with_delimiter()`; builds S3 ListObjectsV2 XML from `ListResult`. `IsTruncated` is always `false`. @@ -52,4 +52,4 @@ cargo test 1. **Multipart uses raw HTTP (S3 only)**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP. They are gated to `backend_type == "s3"` — non-S3 backends return an error for multipart requests and should use `PUT` (object_store handles chunking internally). 2. **LIST returns all results**: `object_store::list_with_delimiter()` fetches all pages internally. No S3-style pagination (continuation tokens, max-keys truncation). `IsTruncated` is always `false`. -3. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `s3-proxy-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. +3. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. diff --git a/Cargo.lock b/Cargo.lock index ce5a537..79b31a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2269,143 +2269,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "s3-proxy-cf-workers" -version = "1.0.4" -dependencies = [ - "async-trait", - "axum", - "bytes", - "chrono", - "console_error_panic_hook", - "futures", - "getrandom 0.2.17", - "getrandom 0.3.4", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "js-sys", - "object_store", - "quick-xml 0.37.5", - "reqwest", - "s3-proxy-core", - "s3-proxy-source-coop", - "s3-proxy-sts", - "serde", - "serde_json", - "thiserror", - "tracing", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "worker", -] - -[[package]] -name = "s3-proxy-core" -version = "1.0.4" -dependencies = [ - "async-trait", - "aws-sdk-dynamodb", - "axum", - "base64", - "bytes", - "chrono", - "futures", - "hex", - "hmac", - "http 1.4.0", - "object_store", - "quick-xml 0.37.5", - "reqwest", - "serde", - "serde_json", - "sha2", - "sqlx", - "thiserror", - "tokio", - "toml", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "s3-proxy-oidc-provider" -version = "1.0.4" -dependencies = [ - "async-trait", - "base64", - "chrono", - "rand 0.8.5", - "rsa", - "s3-proxy-core", - "serde", - "serde_json", - "sha2", - "thiserror", - "tokio", - "tracing", - "uuid", -] - -[[package]] -name = "s3-proxy-server" -version = "1.0.4" -dependencies = [ - "axum", - "bytes", - "futures", - "http 1.4.0", - "http-body-util", - "object_store", - "reqwest", - "s3-proxy-core", - "s3-proxy-source-coop", - "s3-proxy-sts", - "serde", - "thiserror", - "tokio", - "toml", - "tower-service", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "s3-proxy-source-coop" -version = "1.0.4" -dependencies = [ - "bytes", - "http 1.4.0", - "s3-proxy-core", - "serde", - "serde_json", - "tracing", - "url", -] - -[[package]] -name = "s3-proxy-sts" -version = "1.0.4" -dependencies = [ - "async-trait", - "base64", - "chrono", - "quick-xml 0.37.5", - "reqwest", - "rsa", - "s3-proxy-core", - "serde", - "serde_json", - "sha2", - "thiserror", - "tracing", - "url", - "uuid", -] - [[package]] name = "schannel" version = "0.1.28" @@ -2638,6 +2501,52 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "source-coop-api" +version = "1.0.4" +dependencies = [ + "bytes", + "http 1.4.0", + "serde", + "serde_json", + "source-coop-core", + "tracing", + "url", +] + +[[package]] +name = "source-coop-cf-workers" +version = "1.0.4" +dependencies = [ + "async-trait", + "axum", + "bytes", + "chrono", + "console_error_panic_hook", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "js-sys", + "object_store", + "quick-xml 0.37.5", + "reqwest", + "serde", + "serde_json", + "source-coop-api", + "source-coop-core", + "source-coop-sts", + "thiserror", + "tracing", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "worker", +] + [[package]] name = "source-coop-cli" version = "1.0.4" @@ -2655,6 +2564,97 @@ dependencies = [ "url", ] +[[package]] +name = "source-coop-core" +version = "1.0.4" +dependencies = [ + "async-trait", + "aws-sdk-dynamodb", + "axum", + "base64", + "bytes", + "chrono", + "futures", + "hex", + "hmac", + "http 1.4.0", + "object_store", + "quick-xml 0.37.5", + "reqwest", + "serde", + "serde_json", + "sha2", + "sqlx", + "thiserror", + "tokio", + "toml", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "source-coop-oidc-provider" +version = "1.0.4" +dependencies = [ + "async-trait", + "base64", + "chrono", + "rand 0.8.5", + "rsa", + "serde", + "serde_json", + "sha2", + "source-coop-core", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "source-coop-server" +version = "1.0.4" +dependencies = [ + "axum", + "bytes", + "futures", + "http 1.4.0", + "http-body-util", + "object_store", + "reqwest", + "serde", + "source-coop-api", + "source-coop-core", + "source-coop-sts", + "thiserror", + "tokio", + "toml", + "tower-service", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "source-coop-sts" +version = "1.0.4" +dependencies = [ + "async-trait", + "base64", + "chrono", + "quick-xml 0.37.5", + "reqwest", + "rsa", + "serde", + "serde_json", + "sha2", + "source-coop-core", + "thiserror", + "tracing", + "url", + "uuid", +] + [[package]] name = "spin" version = "0.9.8" diff --git a/Cargo.toml b/Cargo.toml index 8b2ee34..2e7137b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,19 +3,19 @@ members = [ "crates/libs/core", "crates/libs/sts", "crates/libs/oidc-provider", - "crates/libs/source-coop", + "crates/libs/api", "crates/runtimes/server", "crates/runtimes/cf-workers", "crates/cli", ] # Worker crate is excluded from default builds because it contains !Send # WASM types that only compile correctly for wasm32 targets. Build it -# separately via: cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +# separately via: cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown default-members = [ "crates/libs/core", "crates/libs/sts", "crates/libs/oidc-provider", - "crates/libs/source-coop", + "crates/libs/api", "crates/runtimes/server", "crates/cli", ] @@ -77,7 +77,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Internal crates -s3-proxy-core = { path = "crates/libs/core" } -s3-proxy-sts = { path = "crates/libs/sts" } -s3-proxy-oidc-provider = { path = "crates/libs/oidc-provider" } -s3-proxy-source-coop = { path = "crates/libs/source-coop" } +source-coop-core = { path = "crates/libs/core" } +source-coop-sts = { path = "crates/libs/sts" } +source-coop-oidc-provider = { path = "crates/libs/oidc-provider" } +source-coop-api = { path = "crates/libs/api" } diff --git a/Dockerfile b/Dockerfile index 51226ad..64ddcf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,15 +3,15 @@ FROM rust:1.82-slim AS builder WORKDIR /app COPY . . -RUN cargo build --release --package s3-proxy-server --bin s3-proxy +RUN cargo build --release --package source-coop-server --bin source-coop-proxy FROM debian:bookworm-slim RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/s3-proxy /usr/local/bin/s3-proxy +COPY --from=builder /app/target/release/source-coop-proxy /usr/local/bin/source-coop-proxy EXPOSE 8080 -ENTRYPOINT ["s3-proxy"] -CMD ["--config", "/etc/s3-proxy/config.toml", "--listen", "0.0.0.0:8080"] +ENTRYPOINT ["source-coop-proxy"] +CMD ["--config", "/etc/source-coop-proxy/config.toml", "--listen", "0.0.0.0:8080"] diff --git a/Makefile b/Makefile index c5ed2ef..088fb83 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ check: cargo check - cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown + cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown fmt: cargo fmt -- --check @@ -18,7 +18,7 @@ test: cargo test run\:server: - cargo run -p s3-proxy-server -- $(ARGS) + cargo run -p source-coop-server -- $(ARGS) run\:workers: npx wrangler dev --cwd crates/runtimes/cf-workers diff --git a/README.md b/README.md index d658b03..5ab3176 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# s3-proxy-rs +# source data proxy A multi-runtime S3 gateway that streams requests to and from backing object stores (S3, MinIO, R2, etc.), providing a unified S3-compliant API with configurable authentication and authorization. @@ -8,8 +8,9 @@ A multi-runtime S3 gateway that streams requests to and from backing object stor flowchart Clients["S3 Clients
(aws cli, boto3, sdk, etc.)"] - subgraph Proxy["s3-proxy-rs"] - Auth["Auth
(SigV4, STS, OIDC)"] + subgraph Proxy["source-coop-proxy"] + Auth["Auth
(STS, OIDC)"] + Core["Core
(SigV4, Proxy)"] Config["Config
(Static, HTTP API, DynamoDB, Postgres)"] end @@ -24,12 +25,15 @@ flowchart ```sh crates/ -├── libs/ # Libraries — not directly runnable -│ ├── core/ (s3-proxy-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config providers -│ └── sts/ (s3-proxy-sts) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) -└── runtimes/ # Runnable targets — one per deployment platform - ├── server/ (s3-proxy-server) # Tokio/Hyper for container deployments - └── cf-workers/ (s3-proxy-cf-workers) # Cloudflare Workers for edge deployments +├── cli/ # source-coop CLI (OIDC login → STS credential exchange) +├── libs/ # Libraries — not directly runnable +│ ├── core/ (source-coop-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config providers +│ ├── sts/ (source-coop-sts) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) +│ ├── oidc-provider/ # Outbound OIDC provider (JWT signing, JWKS, credential exchange) +│ └── source-coop/ # Source Cooperative resolver and API client +└── runtimes/ # Runnable targets — one per deployment platform + ├── server/ (source-coop-server) # Tokio/Hyper for container deployments + └── cf-workers/ (source-coop-cf-workers) # Cloudflare Workers for edge deployments ``` Libraries define trait abstractions (`ProxyBackend`, `ConfigProvider`, `RequestResolver`). Runtimes implement `ProxyBackend` with platform-native primitives. The handler uses a two-phase dispatch model: `resolve_request()` returns a `HandlerAction` — either a `Forward` (presigned URL for GET/HEAD/PUT/DELETE), a `Response` (LIST, errors), or `NeedsBody` (multipart). Runtimes execute `Forward` requests with their native HTTP client, enabling zero-copy streaming. @@ -61,7 +65,7 @@ This starts MinIO (`:9000` API, `:9001` console) and a seed job that creates exa ```bash # Option A: native server runtime -cargo run -p s3-proxy-server -- --config config.local.toml --listen 0.0.0.0:8080 +cargo run -p source-coop-server -- --config config.local.toml --listen 0.0.0.0:8080 # Option B: Cloudflare Workers runtime (via Wrangler) cd crates/runtimes/cf-workers && npx wrangler dev @@ -89,14 +93,14 @@ The server runtime reads `config.local.toml` (TOML, backend endpoints use `http: ```bash # Build -cargo build --release -p s3-proxy-server +cargo build --release -p source-coop-server # Run with a config file -./target/release/s3-proxy --config config.toml --listen 0.0.0.0:8080 +./target/release/source-coop-proxy --config config.toml --listen 0.0.0.0:8080 # Or with Docker -docker build -t s3-proxy . -docker run -v ./config.toml:/etc/s3-proxy/config.toml -p 8080:8080 s3-proxy +docker build -t source-coop-proxy . +docker run -v ./config.toml:/etc/source-coop-proxy/config.toml -p 8080:8080 source-coop-proxy ``` ### Client Usage @@ -135,7 +139,7 @@ The proxy supports multiple backends for retrieving configuration, selectable at #### Static File (always available) ```rust -use s3_proxy_core::config::static_file::StaticProvider; +use source_coop_core::config::static_file::StaticProvider; let provider = StaticProvider::from_file("config.toml")?; // or @@ -149,7 +153,7 @@ let provider = StaticProvider::from_json(&json_string)?; Fetches config from a centralized REST API. Useful with a control plane service. ```rust -use s3_proxy_core::config::http::HttpProvider; +use source_coop_core::config::http::HttpProvider; let provider = HttpProvider::new( "https://config-api.internal:8080".to_string(), @@ -164,24 +168,24 @@ Expected endpoints: `GET /buckets`, `GET /buckets/{name}`, `GET /roles/{id}`, et Single-table design with PK/SK pattern. Build with: ```bash -cargo build -p s3-proxy-server --features s3-proxy-core/config-dynamodb +cargo build -p source-coop-server --features source-coop-core/config-dynamodb ``` ```rust -use s3_proxy_core::config::dynamodb::DynamoDbProvider; +use source_coop_core::config::dynamodb::DynamoDbProvider; let client = aws_sdk_dynamodb::Client::new(&aws_config); -let provider = DynamoDbProvider::new(client, "s3-proxy-config".to_string()); +let provider = DynamoDbProvider::new(client, "source-coop-proxy-config".to_string()); ``` #### PostgreSQL (`config-postgres` feature) ```bash -cargo build -p s3-proxy-server --features s3-proxy-core/config-postgres +cargo build -p source-coop-server --features source-coop-core/config-postgres ``` ```rust -use s3_proxy_core::config::postgres::PostgresProvider; +use source_coop_core::config::postgres::PostgresProvider; let pool = sqlx::PgPool::connect("postgres://localhost/s3proxy").await?; let provider = PostgresProvider::new(pool); @@ -192,9 +196,9 @@ let provider = PostgresProvider::new(pool); Implement the `ConfigProvider` trait to plug in your own config backend. Then wrap it in `DefaultResolver` to get standard S3 proxy behavior (path/virtual-host parsing, auth, authorization): ```rust -use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::*; +use source_coop_core::config::ConfigProvider; +use source_coop_core::error::ProxyError; +use source_coop_core::types::*; #[derive(Clone)] struct MyProvider { /* ... */ } @@ -214,9 +218,9 @@ impl ConfigProvider for MyProvider { For full control over request routing, authentication, and namespace mapping, implement the `RequestResolver` trait directly. This is useful when your URL namespace doesn't map to a simple bucket/key structure, or when authorization is handled by an external service. ```rust -use s3_proxy_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::BucketConfig; +use source_coop_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; +use source_coop_core::error::ProxyError; +use source_coop_core::types::BucketConfig; use http::{Method, HeaderMap}; use bytes::Bytes; @@ -257,7 +261,7 @@ let action = handler.resolve_request(method, path, query, &headers).await; Wrap any provider with `CachedProvider` to add in-memory TTL-based caching. This is recommended for all network-backed providers. ```rust -use s3_proxy_core::config::cached::CachedProvider; +use source_coop_core::config::cached::CachedProvider; use std::time::Duration; // Wrap any provider with a 5-minute cache @@ -274,6 +278,58 @@ provider.invalidate_bucket("my-bucket"); The cache is thread-safe (`RwLock`-based) and evicts entries lazily on access. Temporary credential writes and reads always go directly to the underlying provider — they're already short-lived and caching them would create security issues with stale session tokens. +### Roles + +Roles define trust policies for OIDC token exchange via `AssumeRoleWithWebIdentity`. Each role specifies which OIDC issuers to trust, optional audience and subject constraints, and the access scopes granted to minted credentials. + +```toml +[[roles]] +role_id = "github-actions-deployer" +name = "GitHub Actions Deploy Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +required_audience = "sts.s3proxy.example.com" +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", + "repo:myorg/infrastructure:*", +] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] +actions = ["get_object", "head_object", "put_object"] +``` + +| Field | Description | +| --------------------------- | ------------------------------------------------------------------------ | +| `role_id` | Identifier used as the `RoleArn` in STS requests | +| `trusted_oidc_issuers` | OIDC provider URLs whose tokens are accepted | +| `required_audience` | If set, the token's `aud` claim must match | +| `subject_conditions` | Glob patterns matched against the `sub` claim (`*` matches any sequence) | +| `max_session_duration_secs` | Upper bound for session duration (clients can request less) | +| `allowed_scopes` | List of `{ bucket, prefixes, actions }` granted to minted credentials | + +#### Template Variables in Access Scopes + +Access scope `bucket` and `prefixes` values support `{claim_name}` template variables that are resolved against the authenticated user's JWT claims at credential mint time. This enables per-user bucket access without defining a separate role for each user. + +```toml +[[roles]] +role_id = "source-coop-user" +name = "Source Cooperative User" +trusted_oidc_issuers = ["https://auth.source.coop"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +# Each user gets read/write access to a bucket matching their OIDC subject +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +A user with `sub = "alice"` receives credentials scoped to `bucket = "alice"`. Any JWT string claim can be referenced (e.g., `{email}`, `{org}`). Missing or non-string claims resolve to an empty string, which will fail authorization safely. + ## Authentication Flows ### 1. Anonymous Access @@ -317,11 +373,11 @@ The proxy validates the JWT against the OIDC provider's JWKS, checks the trust p The crate workspace separates concerns so the core logic compiles to both native and WASM targets: -**`s3-proxy-core`** has zero runtime dependencies. No `tokio`, no `worker`. It uses `object_store`'s `Signer` trait to generate presigned URLs for GET/HEAD/PUT/DELETE, and `object_store` directly for LIST. A `ProxyBackend` trait provides runtime-specific store/signer creation and raw HTTP (multipart). All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. +**`source-coop-core`** has zero runtime dependencies. No `tokio`, no `worker`. It uses `object_store`'s `Signer` trait to generate presigned URLs for GET/HEAD/PUT/DELETE, and `object_store` directly for LIST. A `ProxyBackend` trait provides runtime-specific store/signer creation and raw HTTP (multipart). All async trait methods use `impl Future` (RPITIT) rather than `#[async_trait]` with `Send` bounds, so they compile on single-threaded WASM runtimes. -**`s3-proxy-server`** adds Tokio, Hyper, and reqwest. It handles `Forward` actions by executing presigned URLs via reqwest — streaming the Hyper `Incoming` body for PUT and the reqwest `bytes_stream()` for GET responses. No buffering. Multipart and LIST go through the handler's `Response` path. +**`source-coop-server`** adds Tokio, Hyper, and reqwest. It handles `Forward` actions by executing presigned URLs via reqwest — streaming the Hyper `Incoming` body for PUT and the reqwest `bytes_stream()` for GET responses. No buffering. Multipart and LIST go through the handler's `Response` path. -**`s3-proxy-cf-workers`** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It handles `Forward` actions by passing JS `ReadableStream` bodies directly through the Fetch API — zero Rust stream involvement. `FetchConnector` bridges `object_store` to the Workers Fetch API (used only for LIST). +**`source-coop-cf-workers`** adds `worker-rs`, `wasm-bindgen`, and `web-sys`. It handles `Forward` actions by passing JS `ReadableStream` bodies directly through the Fetch API — zero Rust stream involvement. `FetchConnector` bridges `object_store` to the Workers Fetch API (used only for LIST). ## License diff --git a/crates/libs/source-coop/Cargo.toml b/crates/libs/api/Cargo.toml similarity index 84% rename from crates/libs/source-coop/Cargo.toml rename to crates/libs/api/Cargo.toml index 50050d0..afe4239 100644 --- a/crates/libs/source-coop/Cargo.toml +++ b/crates/libs/api/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "s3-proxy-source-coop" +name = "source-coop-api" version.workspace = true edition.workspace = true license.workspace = true description = "Source Cooperative API client and request resolver for the S3 proxy gateway" [dependencies] -s3-proxy-core.workspace = true +source-coop-core.workspace = true serde.workspace = true serde_json.workspace = true http.workspace = true diff --git a/crates/libs/source-coop/src/api.rs b/crates/libs/api/src/api.rs similarity index 98% rename from crates/libs/source-coop/src/api.rs rename to crates/libs/api/src/api.rs index 68c420d..d0201bb 100644 --- a/crates/libs/source-coop/src/api.rs +++ b/crates/libs/api/src/api.rs @@ -4,8 +4,8 @@ //! API keys, and permissions. The actual HTTP transport is abstracted behind //! the [`HttpClient`] trait so each runtime can provide its own implementation. -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::maybe_send::{MaybeSend, MaybeSync}; +use source_coop_core::error::ProxyError; +use source_coop_core::maybe_send::{MaybeSend, MaybeSync}; use serde::de::DeserializeOwned; use serde::Deserialize; use std::collections::HashMap; diff --git a/crates/libs/source-coop/src/lib.rs b/crates/libs/api/src/lib.rs similarity index 100% rename from crates/libs/source-coop/src/lib.rs rename to crates/libs/api/src/lib.rs diff --git a/crates/libs/source-coop/src/resolver.rs b/crates/libs/api/src/resolver.rs similarity index 98% rename from crates/libs/source-coop/src/resolver.rs rename to crates/libs/api/src/resolver.rs index 951b9b6..ae4f5d4 100644 --- a/crates/libs/source-coop/src/resolver.rs +++ b/crates/libs/api/src/resolver.rs @@ -7,10 +7,10 @@ use crate::api::{HttpClient, SourceApiClient}; use bytes::Bytes; use http::{HeaderMap, Method}; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::resolver::{ListRewrite, RequestResolver, ResolvedAction}; -use s3_proxy_core::s3::request::build_s3_operation; -use s3_proxy_core::types::BucketConfig; +use source_coop_core::error::ProxyError; +use source_coop_core::resolver::{ListRewrite, RequestResolver, ResolvedAction}; +use source_coop_core::s3::request::build_s3_operation; +use source_coop_core::types::BucketConfig; use std::collections::HashMap; /// Request resolver for Source Cooperative. @@ -160,7 +160,7 @@ impl SourceCoopResolver { None => return Ok(()), // Anonymous — skip permission check }; - let sig = s3_proxy_core::auth::parse_sigv4_auth(auth_header)?; + let sig = source_coop_core::auth::parse_sigv4_auth(auth_header)?; let perms = self .api_client diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml index 171795d..75e7db8 100644 --- a/crates/libs/core/Cargo.toml +++ b/crates/libs/core/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-core" +name = "source-coop-core" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md index 359e1f6..bb59c2e 100644 --- a/crates/libs/core/README.md +++ b/crates/libs/core/README.md @@ -1,4 +1,4 @@ -# s3-proxy-core +# data.source.coop Runtime-agnostic core library for the S3 proxy gateway. This crate contains all business logic — S3 request parsing, SigV4 signing/verification, authorization, configuration retrieval, and the proxy handler — without depending on any async runtime. @@ -53,16 +53,16 @@ src/ ## Usage -This crate is not used directly. Runtime crates (`s3-proxy-server`, `s3-proxy-cf-workers`) depend on it and provide concrete `ProxyBackend` implementations. If you're building a custom runtime integration, depend on this crate and implement `ProxyBackend`, and optionally `ConfigProvider` or `RequestResolver`. +This crate is not used directly. Runtime crates (`source-coop-server`, `source-coop-cf-workers`) depend on it and provide concrete `ProxyBackend` implementations. If you're building a custom runtime integration, depend on this crate and implement `ProxyBackend`, and optionally `ConfigProvider` or `RequestResolver`. ### Standard usage with a ConfigProvider Wrap your config provider in `DefaultResolver` for standard S3 proxy behavior (path/virtual-host parsing, SigV4 auth, scope-based authorization): ```rust -use s3_proxy_core::proxy::ProxyHandler; -use s3_proxy_core::resolver::DefaultResolver; -use s3_proxy_core::config::static_file::StaticProvider; +use source_coop_core::proxy::ProxyHandler; +use source_coop_core::resolver::DefaultResolver; +use source_coop_core::config::static_file::StaticProvider; let backend = MyBackend::new(); let config = StaticProvider::from_file("config.toml")?; @@ -80,8 +80,8 @@ let action = handler.resolve_request(method, path, query, &headers).await; For non-standard URL namespaces or external auth, implement `RequestResolver` directly: ```rust -use s3_proxy_core::resolver::{RequestResolver, ResolvedAction}; -use s3_proxy_core::error::ProxyError; +use source_coop_core::resolver::{RequestResolver, ResolvedAction}; +use source_coop_core::error::ProxyError; #[derive(Clone)] struct MyResolver { /* ... */ } diff --git a/crates/libs/core/src/config/cached.rs b/crates/libs/core/src/config/cached.rs index ede9fb2..6e0e28e 100644 --- a/crates/libs/core/src/config/cached.rs +++ b/crates/libs/core/src/config/cached.rs @@ -7,7 +7,7 @@ //! # Example //! //! ```rust,ignore -//! use s3_proxy_core::config::cached::CachedProvider; +//! use source_coop_core::config::cached::CachedProvider; //! use std::time::Duration; //! //! // Wrap any provider with a 5-minute cache diff --git a/crates/libs/core/src/config/dynamodb.rs b/crates/libs/core/src/config/dynamodb.rs index fc5c0a2..01171dc 100644 --- a/crates/libs/core/src/config/dynamodb.rs +++ b/crates/libs/core/src/config/dynamodb.rs @@ -17,7 +17,7 @@ //! # Example //! //! ```rust,ignore -//! use s3_proxy_core::config::dynamodb::DynamoDbProvider; +//! use source_coop_core::config::dynamodb::DynamoDbProvider; //! use aws_sdk_dynamodb::Client; //! //! let sdk_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; diff --git a/crates/libs/core/src/config/http.rs b/crates/libs/core/src/config/http.rs index 56ffefe..2408217 100644 --- a/crates/libs/core/src/config/http.rs +++ b/crates/libs/core/src/config/http.rs @@ -16,7 +16,7 @@ //! # Example //! //! ```rust,ignore -//! use s3_proxy_core::config::http::HttpProvider; +//! use source_coop_core::config::http::HttpProvider; //! //! let provider = HttpProvider::new( //! "https://config-api.internal:8080".to_string(), diff --git a/crates/libs/core/src/config/mod.rs b/crates/libs/core/src/config/mod.rs index 35b1ec1..e300759 100644 --- a/crates/libs/core/src/config/mod.rs +++ b/crates/libs/core/src/config/mod.rs @@ -21,7 +21,7 @@ //! network calls (HTTP, DynamoDB, Postgres). //! //! ```rust,ignore -//! use s3_proxy_core::config::{cached::CachedProvider, static_file::StaticProvider}; +//! use source_coop_core::config::{cached::CachedProvider, static_file::StaticProvider}; //! use std::time::Duration; //! //! let base = StaticProvider::from_file("config.toml").unwrap(); diff --git a/crates/libs/core/src/config/postgres.rs b/crates/libs/core/src/config/postgres.rs index d81a6f6..1a57153 100644 --- a/crates/libs/core/src/config/postgres.rs +++ b/crates/libs/core/src/config/postgres.rs @@ -30,7 +30,7 @@ //! # Example //! //! ```rust,ignore -//! use s3_proxy_core::config::postgres::PostgresProvider; +//! use source_coop_core::config::postgres::PostgresProvider; //! use sqlx::PgPool; //! //! let pool = PgPool::connect("postgres://user:pass@localhost/s3proxy").await?; diff --git a/crates/libs/core/src/maybe_send.rs b/crates/libs/core/src/maybe_send.rs index 9fae035..fd3b71a 100644 --- a/crates/libs/core/src/maybe_send.rs +++ b/crates/libs/core/src/maybe_send.rs @@ -13,7 +13,7 @@ //! Use `MaybeSend` instead of `Send` in trait bounds throughout the core: //! //! ```rust,ignore -//! use s3_proxy_core::maybe_send::MaybeSend; +//! use source_coop_core::maybe_send::MaybeSend; //! //! pub trait MyTrait: MaybeSend { //! fn do_work(&self) -> impl Future + MaybeSend; diff --git a/crates/libs/oidc-provider/Cargo.toml b/crates/libs/oidc-provider/Cargo.toml index d53aedc..017e435 100644 --- a/crates/libs/oidc-provider/Cargo.toml +++ b/crates/libs/oidc-provider/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-oidc-provider" +name = "source-coop-oidc-provider" version.workspace = true edition.workspace = true license.workspace = true @@ -11,7 +11,7 @@ azure = [] gcp = [] [dependencies] -s3-proxy-core.workspace = true +source-coop-core.workspace = true async-trait.workspace = true thiserror.workspace = true serde.workspace = true diff --git a/crates/libs/oidc-provider/src/exchange/mod.rs b/crates/libs/oidc-provider/src/exchange/mod.rs index 1d280c9..bff9d6a 100644 --- a/crates/libs/oidc-provider/src/exchange/mod.rs +++ b/crates/libs/oidc-provider/src/exchange/mod.rs @@ -15,12 +15,12 @@ use crate::{CloudCredentials, HttpExchange, OidcProviderError}; /// - Azure: Federated token exchange via Azure AD /// - GCP: STS token exchange + `generateAccessToken` via IAM pub trait CredentialExchange: - s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync + source_coop_core::maybe_send::MaybeSend + source_coop_core::maybe_send::MaybeSync { fn exchange( &self, http: &H, jwt: &str, ) -> impl std::future::Future> - + s3_proxy_core::maybe_send::MaybeSend; + + source_coop_core::maybe_send::MaybeSend; } diff --git a/crates/libs/oidc-provider/src/lib.rs b/crates/libs/oidc-provider/src/lib.rs index d2e9b8e..67b6816 100644 --- a/crates/libs/oidc-provider/src/lib.rs +++ b/crates/libs/oidc-provider/src/lib.rs @@ -38,14 +38,14 @@ pub struct CloudCredentials { /// Each runtime provides its own implementation — `reqwest` on native, /// `Fetch` on Cloudflare Workers. pub trait HttpExchange: - Clone + s3_proxy_core::maybe_send::MaybeSend + s3_proxy_core::maybe_send::MaybeSync + 'static + Clone + source_coop_core::maybe_send::MaybeSend + source_coop_core::maybe_send::MaybeSync + 'static { fn post_form( &self, url: &str, form: &[(&str, &str)], ) -> impl std::future::Future> - + s3_proxy_core::maybe_send::MaybeSend; + + source_coop_core::maybe_send::MaybeSend; } /// Top-level provider that combines signing, exchange, and caching. @@ -122,9 +122,9 @@ pub enum OidcProviderError { HttpError(String), } -impl From for s3_proxy_core::error::ProxyError { +impl From for source_coop_core::error::ProxyError { fn from(e: OidcProviderError) -> Self { - s3_proxy_core::error::ProxyError::Internal(e.to_string()) + source_coop_core::error::ProxyError::Internal(e.to_string()) } } @@ -293,7 +293,7 @@ mod tests { #[test] fn error_converts_to_proxy_error() { let err = OidcProviderError::ExchangeError("test".into()); - let proxy_err: s3_proxy_core::error::ProxyError = err.into(); + let proxy_err: source_coop_core::error::ProxyError = err.into(); assert!(proxy_err.to_string().contains("test")); assert_eq!(proxy_err.status_code(), 500); } diff --git a/crates/libs/sts/Cargo.toml b/crates/libs/sts/Cargo.toml index 02c3593..5096870 100644 --- a/crates/libs/sts/Cargo.toml +++ b/crates/libs/sts/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "s3-proxy-sts" +name = "source-coop-sts" version.workspace = true edition.workspace = true license.workspace = true description = "OIDC/STS authentication for the S3 proxy gateway" [dependencies] -s3-proxy-core.workspace = true +source-coop-core.workspace = true async-trait.workspace = true thiserror.workspace = true serde.workspace = true diff --git a/crates/libs/sts/README.md b/crates/libs/sts/README.md index 19ac2ca..37a4039 100644 --- a/crates/libs/sts/README.md +++ b/crates/libs/sts/README.md @@ -1,4 +1,4 @@ -# s3-proxy-sts +# source-coop-sts OIDC token exchange and STS credential minting for the S3 proxy gateway. Implements the `AssumeRoleWithWebIdentity` flow, allowing workloads like GitHub Actions to exchange OIDC JWTs for temporary, scoped S3 credentials. @@ -10,7 +10,7 @@ GitHub Actions (or any OIDC provider) │ JWT (signed by provider) ▼ ┌─────────────────────────────┐ -│ s3-proxy-sts │ +│ source-coop-sts │ │ │ │ 1. Decode JWT header │ │ 2. Fetch JWKS from issuer │ @@ -53,8 +53,8 @@ src/ Called by the proxy handler when it receives an STS `AssumeRoleWithWebIdentity` request: ```rust -use s3_proxy_sts::assume_role_with_web_identity; -use s3_proxy_sts::request::{StsRequest, try_parse_sts_request}; +use source_coop_sts::assume_role_with_web_identity; +use source_coop_sts::request::{StsRequest, try_parse_sts_request}; // Parse from query string let sts_request = try_parse_sts_request(Some(query)) diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index 4992335..f094acf 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -10,8 +10,8 @@ use base64::Engine; use rsa::pkcs1v15::VerifyingKey; use rsa::signature::Verifier; use rsa::{BigUint, RsaPublicKey}; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::RoleConfig; +use source_coop_core::error::ProxyError; +use source_coop_core::types::RoleConfig; use serde::Deserialize; use sha2::Sha256; diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index 091d623..d29db3d 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -25,9 +25,9 @@ pub use jwks::JwksCache; pub use request::try_parse_sts_request; use request::StsRequest; pub use responses::{build_sts_error_response, build_sts_response}; -use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::TemporaryCredentials; +use source_coop_core::config::ConfigProvider; +use source_coop_core::error::ProxyError; +use source_coop_core::types::TemporaryCredentials; /// Try to handle an STS request. Returns `Some((status, xml))` if the query /// contained an STS action, or `None` if it wasn't an STS request. diff --git a/crates/libs/sts/src/request.rs b/crates/libs/sts/src/request.rs index 3714bbb..470478f 100644 --- a/crates/libs/sts/src/request.rs +++ b/crates/libs/sts/src/request.rs @@ -2,7 +2,7 @@ //! //! Extracts `AssumeRoleWithWebIdentity` parameters from query strings. -use s3_proxy_core::error::ProxyError; +use source_coop_core::error::ProxyError; /// Parsed STS `AssumeRoleWithWebIdentity` request parameters. #[derive(Debug, Clone)] diff --git a/crates/libs/sts/src/responses.rs b/crates/libs/sts/src/responses.rs index a8d2e40..d808cb0 100644 --- a/crates/libs/sts/src/responses.rs +++ b/crates/libs/sts/src/responses.rs @@ -1,8 +1,8 @@ //! STS XML response serialization. use quick_xml::se::to_string as xml_to_string; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::TemporaryCredentials; +use source_coop_core::error::ProxyError; +use source_coop_core::types::TemporaryCredentials; use serde::Serialize; /// STS AssumeRoleWithWebIdentity response. diff --git a/crates/libs/sts/src/sts.rs b/crates/libs/sts/src/sts.rs index 19e846d..8d7b249 100644 --- a/crates/libs/sts/src/sts.rs +++ b/crates/libs/sts/src/sts.rs @@ -1,7 +1,7 @@ //! STS credential minting. use chrono::{Duration, Utc}; -use s3_proxy_core::types::{AccessScope, RoleConfig, TemporaryCredentials}; +use source_coop_core::types::{AccessScope, RoleConfig, TemporaryCredentials}; use uuid::Uuid; /// Resolve `{claim_name}` template variables in access scopes against JWT claims. @@ -103,7 +103,7 @@ fn rand_byte() -> u8 { #[cfg(test)] mod tests { use super::*; - use s3_proxy_core::types::Action; + use source_coop_core::types::Action; use serde_json::json; fn scope(bucket: &str, prefixes: &[&str], actions: &[Action]) -> AccessScope { diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index 1b97110..fffdb3e 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "s3-proxy-cf-workers" +name = "source-coop-cf-workers" version.workspace = true edition.workspace = true license.workspace = true @@ -9,9 +9,9 @@ description = "Cloudflare Workers runtime for the S3 proxy gateway" crate-type = ["cdylib"] [dependencies] -s3-proxy-core = { workspace = true, features = ["axum"] } -s3-proxy-sts.workspace = true -s3-proxy-source-coop.workspace = true +source-coop-core = { workspace = true, features = ["axum"] } +source-coop-sts.workspace = true +source-coop-api.workspace = true axum = { workspace = true, features = ["json"] } bytes.workspace = true http.workspace = true diff --git a/crates/runtimes/cf-workers/README.md b/crates/runtimes/cf-workers/README.md index dbd0d94..749f13e 100644 --- a/crates/runtimes/cf-workers/README.md +++ b/crates/runtimes/cf-workers/README.md @@ -1,4 +1,4 @@ -# s3-proxy-cf-workers +# source-coop-cf-workers Cloudflare Workers runtime for the S3 proxy gateway. Deploys the proxy to the edge using Cloudflare's global network, using presigned URLs for zero-copy streaming and `object_store` with a custom `FetchConnector` for LIST operations. @@ -11,7 +11,7 @@ Client request -> Pick resolver: - SOURCE_API_URL set? -> SourceCoopResolver (dynamic Source Cooperative backends) - Otherwise -> DefaultResolver (static PROXY_CONFIG) - -> ProxyHandler::resolve_request() (from s3-proxy-core) + -> ProxyHandler::resolve_request() (from source-coop-core) -> Forward: fetch(presigned URL) with ReadableStream passthrough (GET/HEAD/PUT/DELETE) -> Response: LIST XML via object_store, errors, synthetic responses -> NeedsBody: multipart operations via raw signed HTTP @@ -68,8 +68,8 @@ SOURCE_API_URL = "https://api.source.coop" To add a new operating mode, implement `RequestResolver` in a new module: ```rust -use s3_proxy_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; -use s3_proxy_core::error::ProxyError; +use source_coop_core::resolver::{RequestResolver, ResolvedAction, ListRewrite}; +use source_coop_core::error::ProxyError; #[derive(Clone)] struct MyResolver { /* ... */ } @@ -139,5 +139,5 @@ Cloudflare Workers compile to `wasm32-unknown-unknown` and link against `worker- This crate must always be built with `--target wasm32-unknown-unknown`: ```bash -cargo check -p s3-proxy-cf-workers --target wasm32-unknown-unknown +cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown ``` diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 557ef2b..b521102 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -9,12 +9,12 @@ use bytes::Bytes; use http::HeaderMap; use object_store::signer::Signer; use object_store::ObjectStore; -use s3_proxy_core::backend::{ +use source_coop_core::backend::{ build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder, }; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::BucketConfig; -use s3_proxy_source_coop::api::{CacheOptions, HttpClient}; +use source_coop_core::error::ProxyError; +use source_coop_core::types::BucketConfig; +use source_coop_api::api::{CacheOptions, HttpClient}; use serde::de::DeserializeOwned; use std::sync::Arc; use worker::{Cache, Fetch}; diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 16c6c37..32ff668 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -28,13 +28,13 @@ mod fetch_connector; mod tracing_layer; use client::WorkerBackend; -use s3_proxy_core::axum::{build_proxy_response, error_response}; -use s3_proxy_core::config::static_file::{StaticConfig, StaticProvider}; -use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; -use s3_proxy_core::resolver::{DefaultResolver, RequestResolver}; -use s3_proxy_source_coop::api::{CacheTtls, SourceApiClient}; -use s3_proxy_source_coop::resolver::SourceCoopResolver; -use s3_proxy_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; +use source_coop_core::axum::{build_proxy_response, error_response}; +use source_coop_core::config::static_file::{StaticConfig, StaticProvider}; +use source_coop_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; +use source_coop_core::resolver::{DefaultResolver, RequestResolver}; +use source_coop_api::api::{CacheTtls, SourceApiClient}; +use source_coop_api::resolver::SourceCoopResolver; +use source_coop_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; use axum::body::Body; use axum::response::Response; diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index 63920b2..03e59d7 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -1,6 +1,6 @@ compatibility_date = "2024-09-23" main = "build/worker/shim.mjs" -name = "s3-proxy-rs" +name = "source-coop-proxy" [build] command = "cargo install worker-build && worker-build --release" diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index 9427ffa..114150b 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -1,14 +1,14 @@ [package] -name = "s3-proxy-server" +name = "source-coop-server" version.workspace = true edition.workspace = true license.workspace = true description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] -s3-proxy-core = { workspace = true, features = ["axum", "azure", "gcp"] } -s3-proxy-sts.workspace = true -s3-proxy-source-coop.workspace = true +source-coop-core = { workspace = true, features = ["axum", "azure", "gcp"] } +source-coop-sts.workspace = true +source-coop-api.workspace = true axum = { workspace = true, features = ["json", "tokio", "http1", "http2"] } tower-service.workspace = true tokio.workspace = true diff --git a/crates/runtimes/server/README.md b/crates/runtimes/server/README.md index 4242a8d..9f56256 100644 --- a/crates/runtimes/server/README.md +++ b/crates/runtimes/server/README.md @@ -1,4 +1,4 @@ -# s3-proxy-server +# source-coop-server Tokio/Hyper runtime for the S3 proxy gateway. This is the container-deployment crate — it wires the core library into a production HTTP server using native Rust async I/O. @@ -19,32 +19,32 @@ src/ ├── client.rs ServerBackend implementing ProxyBackend ├── server.rs Hyper server setup, two-phase request handling, Forward execution └── bin/ - └── s3-proxy.rs CLI binary entry point + └── source-coop-proxy.rs CLI binary entry point ``` ## Binary Usage ```bash -cargo build --release -p s3-proxy-server +cargo build --release -p source-coop-server # Minimal -./target/release/s3-proxy --config config.toml +./target/release/source-coop-proxy --config config.toml # Full options -./target/release/s3-proxy \ - --config /etc/s3-proxy/config.toml \ +./target/release/source-coop-proxy \ + --config /etc/source-coop-proxy/config.toml \ --listen 0.0.0.0:9000 \ --domain s3.local # Environment variable for log level -RUST_LOG=s3_proxy=debug ./target/release/s3-proxy --config config.toml +RUST_LOG=source_coop=debug ./target/release/source-coop-proxy --config config.toml ``` ## Docker ```bash -docker build -t s3-proxy . -docker run -v ./config.toml:/etc/s3-proxy/config.toml -p 8080:8080 s3-proxy +docker build -t source-coop-proxy . +docker run -v ./config.toml:/etc/source-coop-proxy/config.toml -p 8080:8080 source-coop-proxy ``` ## Using a Different Config Provider @@ -52,9 +52,9 @@ docker run -v ./config.toml:/etc/s3-proxy/config.toml -p 8080:8080 s3-proxy The default binary uses `StaticProvider` (TOML file) wrapped in `CachedProvider`. The `run()` function accepts any `ConfigProvider` and wraps it in a `DefaultResolver` internally. To use a different provider, modify the binary or write your own: ```rust -use s3_proxy_core::config::cached::CachedProvider; -use s3_proxy_core::config::http::HttpProvider; // requires config-http feature -use s3_proxy_server::server::{run, ServerConfig}; +use source_coop_core::config::cached::CachedProvider; +use source_coop_core::config::http::HttpProvider; // requires config-http feature +use source_coop_server::server::{run, ServerConfig}; use std::time::Duration; #[tokio::main] @@ -74,9 +74,9 @@ async fn main() -> Result<(), Box> { For full control over request routing and authorization, you can bypass `run()` and wire up a `ProxyHandler` with a custom `RequestResolver` directly. This is useful when your URL namespace doesn't follow the standard S3 bucket/key pattern, or when authorization is handled by an external service. ```rust -use s3_proxy_core::proxy::ProxyHandler; -use s3_proxy_core::resolver::{RequestResolver, ResolvedAction}; -use s3_proxy_core::error::ProxyError; +use source_coop_core::proxy::ProxyHandler; +use source_coop_core::resolver::{RequestResolver, ResolvedAction}; +use source_coop_core::error::ProxyError; #[derive(Clone)] struct MyResolver { /* ... */ } diff --git a/crates/runtimes/server/src/bin/s3-proxy.rs b/crates/runtimes/server/src/bin/source-coop-proxy.rs similarity index 80% rename from crates/runtimes/server/src/bin/s3-proxy.rs rename to crates/runtimes/server/src/bin/source-coop-proxy.rs index 1a6b75a..cf12946 100644 --- a/crates/runtimes/server/src/bin/s3-proxy.rs +++ b/crates/runtimes/server/src/bin/source-coop-proxy.rs @@ -1,11 +1,11 @@ -//! S3 Proxy Server binary. +//! Source Cooperative Proxy Server binary. //! //! Usage: -//! s3-proxy --config config.toml [--sts-config sts.toml] [--listen 0.0.0.0:8080] [--domain s3.local] +//! source-coop-proxy --config config.toml [--sts-config sts.toml] [--listen 0.0.0.0:8080] [--domain s3.local] -use s3_proxy_core::config::cached::CachedProvider; -use s3_proxy_core::config::static_file::StaticProvider; -use s3_proxy_server::server::{run, ServerConfig}; +use source_coop_core::config::cached::CachedProvider; +use source_coop_core::config::static_file::StaticProvider; +use source_coop_server::server::{run, ServerConfig}; use std::net::SocketAddr; use std::time::Duration; @@ -14,7 +14,7 @@ async fn main() -> Result<(), Box> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "s3_proxy=info".into()), + .unwrap_or_else(|_| "source_coop=info".into()), ) .init(); @@ -46,7 +46,7 @@ async fn main() -> Result<(), Box> { .and_then(|i| args.get(i + 1)) .map(|s| s.as_str()); - tracing::info!(config = %config_path, listen = %listen_addr, "starting s3-proxy"); + tracing::info!(config = %config_path, listen = %listen_addr, "starting source-coop-proxy"); let base_config = StaticProvider::from_file(config_path)?; let sts_base = match sts_config_path { diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index 3d5ad4e..a3338c6 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -4,9 +4,9 @@ use bytes::Bytes; use http::HeaderMap; use object_store::signer::Signer; use object_store::ObjectStore; -use s3_proxy_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse}; -use s3_proxy_core::error::ProxyError; -use s3_proxy_core::types::BucketConfig; +use source_coop_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse}; +use source_coop_core::error::ProxyError; +use source_coop_core::types::BucketConfig; use std::sync::Arc; /// Backend for the Tokio/Hyper server runtime. diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 4be82fb..c99df07 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -8,11 +8,11 @@ use axum::Router; use futures::TryStreamExt; use http::HeaderMap; use http_body_util::BodyStream; -use s3_proxy_core::axum::{build_proxy_response, error_response}; -use s3_proxy_core::config::ConfigProvider; -use s3_proxy_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; -use s3_proxy_core::resolver::DefaultResolver; -use s3_proxy_sts::{try_handle_sts, JwksCache}; +use source_coop_core::axum::{build_proxy_response, error_response}; +use source_coop_core::config::ConfigProvider; +use source_coop_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; +use source_coop_core::resolver::DefaultResolver; +use source_coop_sts::{try_handle_sts, JwksCache}; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; @@ -47,8 +47,8 @@ struct AppState { /// # Example /// /// ```rust,ignore -/// use s3_proxy_core::config::static_file::StaticProvider; -/// use s3_proxy_server::server::{run, ServerConfig}; +/// use source_coop_core::config::static_file::StaticProvider; +/// use source_coop_server::server::{run, ServerConfig}; /// /// #[tokio::main] /// async fn main() { From 2f05f245dc2b2c0958f92aae4c6e76c50f065b8b Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 23:26:18 -0500 Subject: [PATCH 43/82] ci: add rust tooling --- .github/workflows/ci.yaml | 61 +++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index abd82b9..1cf2721 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,18 +4,61 @@ on: push: jobs: + fmt: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + components: rustfmt + - run: cargo fmt --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + components: clippy + - run: cargo clippy -- -D warnings + + check: + name: Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + targets: wasm32-unknown-unknown + - name: Check workspace + run: cargo check + - name: Check cf-workers (wasm32) + run: cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown + test: + name: Test runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + - run: cargo test + build: + name: Build + runs-on: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - name: Set up Rust - uses: actions-rs/toolchain@8e603f32c5c6eeca5b1b2d9d1e7464d926082f1d # v1.0.0 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable - - name: Format - run: cargo fmt --check - - name: Clippy - run: cargo clippy -- -D warnings - - name: Run tests - run: cargo test + - name: Build server + run: cargo build -p source-coop-server + - name: Build CLI + run: cargo build -p source-coop-cli From 7ac0999b3edbffedc0e6263417df4f3e0eb64916 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 23:27:03 -0500 Subject: [PATCH 44/82] chore: trigger ci From ddc4c5715ab0744040da547b35109e813496dd4d Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 23:32:20 -0500 Subject: [PATCH 45/82] chore: cargo fmt --- crates/cli/src/oidc.rs | 5 +- crates/cli/src/sts.rs | 3 +- crates/libs/api/src/api.rs | 4 +- crates/libs/core/src/auth.rs | 44 ++++--- crates/libs/core/src/proxy.rs | 35 +++-- crates/libs/core/src/s3/pagination.rs | 155 +++++++++++++++++------ crates/libs/core/src/s3/response.rs | 5 +- crates/libs/sts/src/jwks.rs | 12 +- crates/libs/sts/src/lib.rs | 5 +- crates/libs/sts/src/responses.rs | 8 +- crates/libs/sts/src/sts.rs | 13 +- crates/runtimes/cf-workers/src/client.rs | 4 +- crates/runtimes/cf-workers/src/lib.rs | 53 +++++--- crates/runtimes/server/src/server.rs | 18 +-- 14 files changed, 238 insertions(+), 126 deletions(-) diff --git a/crates/cli/src/oidc.rs b/crates/cli/src/oidc.rs index 5ff815a..adc94ef 100644 --- a/crates/cli/src/oidc.rs +++ b/crates/cli/src/oidc.rs @@ -25,10 +25,7 @@ pub async fn discover(issuer: &str) -> Result { .map_err(|e| format!("Failed to fetch OIDC discovery document: {e}"))?; if !resp.status().is_success() { - return Err(format!( - "OIDC discovery returned status {}", - resp.status() - )); + return Err(format!("OIDC discovery returned status {}", resp.status())); } let doc: serde_json::Value = resp diff --git a/crates/cli/src/sts.rs b/crates/cli/src/sts.rs index 313ef37..391ebcd 100644 --- a/crates/cli/src/sts.rs +++ b/crates/cli/src/sts.rs @@ -16,8 +16,7 @@ pub async fn assume_role( web_identity_token: &str, duration_seconds: Option, ) -> Result { - let mut url = url::Url::parse(proxy_url) - .map_err(|e| format!("Invalid proxy URL: {e}"))?; + let mut url = url::Url::parse(proxy_url).map_err(|e| format!("Invalid proxy URL: {e}"))?; url.query_pairs_mut() .append_pair("Action", "AssumeRoleWithWebIdentity") diff --git a/crates/libs/api/src/api.rs b/crates/libs/api/src/api.rs index d0201bb..f2c5330 100644 --- a/crates/libs/api/src/api.rs +++ b/crates/libs/api/src/api.rs @@ -4,10 +4,10 @@ //! API keys, and permissions. The actual HTTP transport is abstracted behind //! the [`HttpClient`] trait so each runtime can provide its own implementation. -use source_coop_core::error::ProxyError; -use source_coop_core::maybe_send::{MaybeSend, MaybeSync}; use serde::de::DeserializeOwned; use serde::Deserialize; +use source_coop_core::error::ProxyError; +use source_coop_core::maybe_send::{MaybeSend, MaybeSync}; use std::collections::HashMap; use std::future::Future; diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 73bbf25..7751568 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -191,10 +191,7 @@ pub async fn resolve_identity( .get("x-amz-security-token") .and_then(|v| v.to_str().ok()) .unwrap_or(""); - if !constant_time_eq( - session_token.as_bytes(), - temp_cred.session_token.as_bytes(), - ) { + if !constant_time_eq(session_token.as_bytes(), temp_cred.session_token.as_bytes()) { return Err(ProxyError::AccessDenied); } @@ -330,7 +327,10 @@ mod tests { .find(|c| c.access_key_id == access_key_id) .cloned()) } - async fn store_temporary_credential(&self, _: &TemporaryCredentials) -> Result<(), ProxyError> { + async fn store_temporary_credential( + &self, + _: &TemporaryCredentials, + ) -> Result<(), ProxyError> { Ok(()) } async fn get_temporary_credential( @@ -387,9 +387,11 @@ mod tests { amz_date, credential_scope, canonical_request_hash ); - let k_date = - hmac_sha256(format!("AWS4{}", secret_access_key).as_bytes(), date_stamp.as_bytes()) - .unwrap(); + let k_date = hmac_sha256( + format!("AWS4{}", secret_access_key).as_bytes(), + date_stamp.as_bytes(), + ) + .unwrap(); let k_region = hmac_sha256(&k_date, region.as_bytes()).unwrap(); let k_service = hmac_sha256(&k_region, b"s3").unwrap(); let signing_key = hmac_sha256(&k_service, b"aws4_request").unwrap(); @@ -402,10 +404,7 @@ mod tests { } /// Build headers and auth for a simple GET request. - fn make_signed_headers( - access_key_id: &str, - secret_access_key: &str, - ) -> HeaderMap { + fn make_signed_headers(access_key_id: &str, secret_access_key: &str) -> HeaderMap { let date_stamp = "20240101"; let amz_date = "20240101T000000Z"; let region = "us-east-1"; @@ -587,7 +586,12 @@ mod tests { date_stamp, amz_date, "us-east-1", - &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + &[ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-amz-security-token", + ], payload_hash, ); headers.insert("authorization", auth.parse().unwrap()); @@ -634,7 +638,12 @@ mod tests { date_stamp, amz_date, "us-east-1", - &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + &[ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-amz-security-token", + ], payload_hash, ); headers.insert("authorization", auth.parse().unwrap()); @@ -682,7 +691,12 @@ mod tests { date_stamp, amz_date, "us-east-1", - &["host", "x-amz-content-sha256", "x-amz-date", "x-amz-security-token"], + &[ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-amz-security-token", + ], payload_hash, ); headers.insert("authorization", auth.parse().unwrap()); diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 82aa0aa..3d6f765 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -161,12 +161,7 @@ where s3_code = %err.s3_error_code(), "request failed" ); - HandlerAction::Response(error_response( - &err, - path, - &request_id, - self.debug_errors, - )) + HandlerAction::Response(error_response(&err, path, &request_id, self.debug_errors)) } } } @@ -750,7 +745,10 @@ mod tests { fn test_bucket_config() -> BucketConfig { let mut backend_options = HashMap::new(); - backend_options.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + backend_options.insert( + "endpoint".into(), + "https://s3.us-east-1.amazonaws.com".into(), + ); backend_options.insert("bucket_name".into(), "my-backend-bucket".into()); BucketConfig { name: "test".into(), @@ -778,15 +776,21 @@ mod tests { // The & and = characters in upload_id must be percent-encoded so they // cannot act as query parameter separators/assignments. let query = url.split_once('?').unwrap().1; - let params: Vec<(String, String)> = - url::form_urlencoded::parse(query.as_bytes()) - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); + let params: Vec<(String, String)> = url::form_urlencoded::parse(query.as_bytes()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); // Should be exactly 2 params: partNumber and uploadId - assert_eq!(params.len(), 2, "expected 2 query params, got: {:?}", params); + assert_eq!( + params.len(), + 2, + "expected 2 query params, got: {:?}", + params + ); assert!(params.iter().any(|(k, v)| k == "partNumber" && v == "1")); - assert!(params.iter().any(|(k, v)| k == "uploadId" && v == malicious_upload_id)); + assert!(params + .iter() + .any(|(k, v)| k == "uploadId" && v == malicious_upload_id)); } #[test] @@ -821,6 +825,9 @@ mod tests { assert!(url.starts_with("https://s3.us-east-1.amazonaws.com/my-backend-bucket/file.bin?")); assert!(url.contains("partNumber=3")); - assert!(url.contains("uploadId=2~abcdef1234567890") || url.contains("uploadId=2%7Eabcdef1234567890")); + assert!( + url.contains("uploadId=2~abcdef1234567890") + || url.contains("uploadId=2%7Eabcdef1234567890") + ); } } diff --git a/crates/libs/core/src/s3/pagination.rs b/crates/libs/core/src/s3/pagination.rs index 0b56f8a..d3d765a 100644 --- a/crates/libs/core/src/s3/pagination.rs +++ b/crates/libs/core/src/s3/pagination.rs @@ -28,7 +28,12 @@ pub struct PaginatedList { /// Parse `max-keys`, `continuation-token`, and `start-after` from a query string. pub fn parse_pagination_params(raw_query: Option<&str>) -> PaginationParams { let pairs = url::form_urlencoded::parse(raw_query.unwrap_or("").as_bytes()); - let find = |name| pairs.clone().find(|(k, _)| k == name).map(|(_, v)| v.to_string()); + let find = |name| { + pairs + .clone() + .find(|(k, _)| k == name) + .map(|(_, v)| v.to_string()) + }; PaginationParams { max_keys: find("max-keys") @@ -91,8 +96,11 @@ pub fn paginate( let is_truncated = page.len() > params.max_keys; page.truncate(params.max_keys); - let next_continuation_token = - if is_truncated { page.last().map(|e| B64.encode(e.key())) } else { None }; + let next_continuation_token = if is_truncated { + page.last().map(|e| B64.encode(e.key())) + } else { + None + }; // Split back into contents and common_prefixes let mut result_contents = Vec::new(); @@ -131,7 +139,9 @@ mod tests { fn make_prefixes(prefixes: &[&str]) -> Vec { prefixes .iter() - .map(|p| ListCommonPrefix { prefix: p.to_string() }) + .map(|p| ListCommonPrefix { + prefix: p.to_string(), + }) .collect() } @@ -146,7 +156,10 @@ mod tests { #[test] fn parse_max_keys_clamped_to_1000() { assert_eq!(parse_pagination_params(Some("max-keys=5")).max_keys, 5); - assert_eq!(parse_pagination_params(Some("max-keys=9999")).max_keys, 1000); + assert_eq!( + parse_pagination_params(Some("max-keys=9999")).max_keys, + 1000 + ); assert_eq!(parse_pagination_params(Some("max-keys=abc")).max_keys, 1000); } @@ -162,9 +175,16 @@ mod tests { #[test] fn no_truncation() { - let r = paginate(make_contents(&["a", "b", "c"]), vec![], &PaginationParams { - max_keys: 1000, continuation_token: None, start_after: None, - }).unwrap(); + let r = paginate( + make_contents(&["a", "b", "c"]), + vec![], + &PaginationParams { + max_keys: 1000, + continuation_token: None, + start_after: None, + }, + ) + .unwrap(); assert_eq!(r.contents.len(), 3); assert!(!r.is_truncated); assert!(r.next_continuation_token.is_none()); @@ -172,9 +192,16 @@ mod tests { #[test] fn truncation_and_token() { - let r = paginate(make_contents(&["a", "b", "c", "d", "e"]), vec![], &PaginationParams { - max_keys: 2, continuation_token: None, start_after: None, - }).unwrap(); + let r = paginate( + make_contents(&["a", "b", "c", "d", "e"]), + vec![], + &PaginationParams { + max_keys: 2, + continuation_token: None, + start_after: None, + }, + ) + .unwrap(); assert_eq!(r.contents.len(), 2); assert!(r.is_truncated); assert_eq!(r.contents[0].key, "a"); @@ -185,35 +212,66 @@ mod tests { #[test] fn continuation_token_round_trip() { let items = make_contents(&["a", "b", "c", "d", "e"]); - let mk = |token| PaginationParams { max_keys: 2, continuation_token: token, start_after: None }; + let mk = |token| PaginationParams { + max_keys: 2, + continuation_token: token, + start_after: None, + }; let p1 = paginate(items.clone(), vec![], &mk(None)).unwrap(); - assert_eq!(p1.contents.iter().map(|c| &c.key).collect::>(), &["a", "b"]); + assert_eq!( + p1.contents.iter().map(|c| &c.key).collect::>(), + &["a", "b"] + ); let p2 = paginate(items.clone(), vec![], &mk(p1.next_continuation_token)).unwrap(); - assert_eq!(p2.contents.iter().map(|c| &c.key).collect::>(), &["c", "d"]); + assert_eq!( + p2.contents.iter().map(|c| &c.key).collect::>(), + &["c", "d"] + ); let p3 = paginate(items.clone(), vec![], &mk(p2.next_continuation_token)).unwrap(); - assert_eq!(p3.contents.iter().map(|c| &c.key).collect::>(), &["e"]); + assert_eq!( + p3.contents.iter().map(|c| &c.key).collect::>(), + &["e"] + ); assert!(!p3.is_truncated); } #[test] fn start_after() { - let r = paginate(make_contents(&["a", "b", "c", "d"]), vec![], &PaginationParams { - max_keys: 1000, continuation_token: None, start_after: Some("b".into()), - }).unwrap(); - assert_eq!(r.contents.iter().map(|c| &c.key).collect::>(), &["c", "d"]); + let r = paginate( + make_contents(&["a", "b", "c", "d"]), + vec![], + &PaginationParams { + max_keys: 1000, + continuation_token: None, + start_after: Some("b".into()), + }, + ) + .unwrap(); + assert_eq!( + r.contents.iter().map(|c| &c.key).collect::>(), + &["c", "d"] + ); } #[test] fn continuation_token_overrides_start_after() { - let r = paginate(make_contents(&["a", "b", "c", "d", "e"]), vec![], &PaginationParams { - max_keys: 1000, - continuation_token: Some(B64.encode("c")), - start_after: Some("a".into()), - }).unwrap(); - assert_eq!(r.contents.iter().map(|c| &c.key).collect::>(), &["d", "e"]); + let r = paginate( + make_contents(&["a", "b", "c", "d", "e"]), + vec![], + &PaginationParams { + max_keys: 1000, + continuation_token: Some(B64.encode("c")), + start_after: Some("a".into()), + }, + ) + .unwrap(); + assert_eq!( + r.contents.iter().map(|c| &c.key).collect::>(), + &["d", "e"] + ); } #[test] @@ -221,8 +279,13 @@ mod tests { let r = paginate( make_contents(&["a.txt", "c.txt"]), make_prefixes(&["b/", "d/"]), - &PaginationParams { max_keys: 3, continuation_token: None, start_after: None }, - ).unwrap(); + &PaginationParams { + max_keys: 3, + continuation_token: None, + start_after: None, + }, + ) + .unwrap(); assert_eq!(r.contents.len(), 2); assert_eq!(r.common_prefixes.len(), 1); assert_eq!(r.contents[0].key, "a.txt"); @@ -233,26 +296,46 @@ mod tests { #[test] fn invalid_token_returns_error() { - let r = paginate(make_contents(&["a"]), vec![], &PaginationParams { - max_keys: 1000, continuation_token: Some("not-valid!!!".into()), start_after: None, - }); + let r = paginate( + make_contents(&["a"]), + vec![], + &PaginationParams { + max_keys: 1000, + continuation_token: Some("not-valid!!!".into()), + start_after: None, + }, + ); assert!(r.is_err()); } #[test] fn max_keys_zero() { - let r = paginate(make_contents(&["a", "b"]), vec![], &PaginationParams { - max_keys: 0, continuation_token: None, start_after: None, - }).unwrap(); + let r = paginate( + make_contents(&["a", "b"]), + vec![], + &PaginationParams { + max_keys: 0, + continuation_token: None, + start_after: None, + }, + ) + .unwrap(); assert!(r.contents.is_empty()); assert!(r.is_truncated); } #[test] fn empty_input() { - let r = paginate(vec![], vec![], &PaginationParams { - max_keys: 1000, continuation_token: None, start_after: None, - }).unwrap(); + let r = paginate( + vec![], + vec![], + &PaginationParams { + max_keys: 1000, + continuation_token: None, + start_after: None, + }, + ) + .unwrap(); assert_eq!(r.contents.len(), 0); assert!(!r.is_truncated); } diff --git a/crates/libs/core/src/s3/response.rs b/crates/libs/core/src/s3/response.rs index 08d7c15..43143df 100644 --- a/crates/libs/core/src/s3/response.rs +++ b/crates/libs/core/src/s3/response.rs @@ -176,7 +176,10 @@ pub struct ListBucketResult { pub start_after: Option, #[serde(rename = "ContinuationToken", skip_serializing_if = "Option::is_none")] pub continuation_token: Option, - #[serde(rename = "NextContinuationToken", skip_serializing_if = "Option::is_none")] + #[serde( + rename = "NextContinuationToken", + skip_serializing_if = "Option::is_none" + )] pub next_continuation_token: Option, #[serde(rename = "Contents", default)] pub contents: Vec, diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index f094acf..3419a77 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -10,10 +10,10 @@ use base64::Engine; use rsa::pkcs1v15::VerifyingKey; use rsa::signature::Verifier; use rsa::{BigUint, RsaPublicKey}; -use source_coop_core::error::ProxyError; -use source_coop_core::types::RoleConfig; use serde::Deserialize; use sha2::Sha256; +use source_coop_core::error::ProxyError; +use source_coop_core::types::RoleConfig; #[derive(Debug, Clone, Deserialize)] pub struct JwksResponse { @@ -185,7 +185,9 @@ pub fn verify_token( if let Some(nbf) = claims.get("nbf").and_then(|v| v.as_i64()) { if now < nbf - CLOCK_SKEW_SECS { - return Err(ProxyError::InvalidOidcToken("token is not yet valid".into())); + return Err(ProxyError::InvalidOidcToken( + "token is not yet valid".into(), + )); } } @@ -235,9 +237,7 @@ impl JwksCache { fn get_cached(&self, issuer: &str) -> Option { let entries = self.entries.lock().unwrap(); if let Some((fetched_at, jwks)) = entries.get(issuer) { - let elapsed = Utc::now() - .signed_duration_since(*fetched_at) - .num_seconds(); + let elapsed = Utc::now().signed_duration_since(*fetched_at).num_seconds(); if elapsed >= 0 && (elapsed as u64) < self.ttl.as_secs() { return Some(jwks.clone()); } diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index d29db3d..fb2c032 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -105,10 +105,7 @@ pub async fn assume_role_with_web_identity( } // Fail fast on unsupported algorithms before making any network requests - let alg = header - .get("alg") - .and_then(|v| v.as_str()) - .unwrap_or(""); + let alg = header.get("alg").and_then(|v| v.as_str()).unwrap_or(""); if alg != "RS256" { return Err(ProxyError::InvalidOidcToken(format!( "unsupported JWT algorithm: {}", diff --git a/crates/libs/sts/src/responses.rs b/crates/libs/sts/src/responses.rs index d808cb0..e80c54a 100644 --- a/crates/libs/sts/src/responses.rs +++ b/crates/libs/sts/src/responses.rs @@ -1,9 +1,9 @@ //! STS XML response serialization. use quick_xml::se::to_string as xml_to_string; +use serde::Serialize; use source_coop_core::error::ProxyError; use source_coop_core::types::TemporaryCredentials; -use serde::Serialize; /// STS AssumeRoleWithWebIdentity response. #[derive(Debug, Serialize)] @@ -72,7 +72,11 @@ pub fn build_sts_response(creds: &TemporaryCredentials) -> (u16, String) { /// Build an STS error response (status code + XML body) from a ProxyError. pub fn build_sts_error_response(err: &ProxyError) -> (u16, String) { let (status, code, message) = match err { - ProxyError::RoleNotFound(r) => (400, "MalformedPolicyDocument", format!("role not found: {}", r)), + ProxyError::RoleNotFound(r) => ( + 400, + "MalformedPolicyDocument", + format!("role not found: {}", r), + ), ProxyError::InvalidOidcToken(msg) => (400, "InvalidIdentityToken", msg.clone()), ProxyError::InvalidRequest(msg) => (400, "InvalidParameterValue", msg.clone()), ProxyError::AccessDenied => (403, "AccessDenied", "access denied".to_string()), diff --git a/crates/libs/sts/src/sts.rs b/crates/libs/sts/src/sts.rs index 8d7b249..5c65386 100644 --- a/crates/libs/sts/src/sts.rs +++ b/crates/libs/sts/src/sts.rs @@ -36,10 +36,7 @@ fn resolve_template(template: &str, claims: &serde_json::Value) -> String { if let Some(end) = result[start..].find('}') { let end = start + end; let key = &result[start + 1..end]; - let value = claims - .get(key) - .and_then(|v| v.as_str()) - .unwrap_or(""); + let value = claims.get(key).and_then(|v| v.as_str()).unwrap_or(""); result = format!("{}{}{}", &result[..start], value, &result[end + 1..]); } else { break; @@ -103,8 +100,8 @@ fn rand_byte() -> u8 { #[cfg(test)] mod tests { use super::*; - use source_coop_core::types::Action; use serde_json::json; + use source_coop_core::types::Action; fn scope(bucket: &str, prefixes: &[&str], actions: &[Action]) -> AccessScope { AccessScope { @@ -150,7 +147,11 @@ mod tests { #[test] fn missing_claim_resolves_to_empty() { - let scopes = vec![scope("{missing}", &["{also_missing}/"], &[Action::GetObject])]; + let scopes = vec![scope( + "{missing}", + &["{also_missing}/"], + &[Action::GetObject], + )]; let claims = json!({"sub": "alice"}); let resolved = resolve_scopes(&scopes, &claims); assert_eq!(resolved[0].bucket, ""); diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index b521102..ef57d63 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -9,13 +9,13 @@ use bytes::Bytes; use http::HeaderMap; use object_store::signer::Signer; use object_store::ObjectStore; +use serde::de::DeserializeOwned; +use source_coop_api::api::{CacheOptions, HttpClient}; use source_coop_core::backend::{ build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder, }; use source_coop_core::error::ProxyError; use source_coop_core::types::BucketConfig; -use source_coop_api::api::{CacheOptions, HttpClient}; -use serde::de::DeserializeOwned; use std::sync::Arc; use worker::{Cache, Fetch}; diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 32ff668..f9690b1 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -28,12 +28,14 @@ mod fetch_connector; mod tracing_layer; use client::WorkerBackend; +use source_coop_api::api::{CacheTtls, SourceApiClient}; +use source_coop_api::resolver::SourceCoopResolver; use source_coop_core::axum::{build_proxy_response, error_response}; use source_coop_core::config::static_file::{StaticConfig, StaticProvider}; -use source_coop_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; +use source_coop_core::proxy::{ + ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, +}; use source_coop_core::resolver::{DefaultResolver, RequestResolver}; -use source_coop_api::api::{CacheTtls, SourceApiClient}; -use source_coop_api::resolver::SourceCoopResolver; use source_coop_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; use axum::body::Body; @@ -78,9 +80,7 @@ async fn fetch( // STS uses STS_CONFIG (falling back to PROXY_CONFIG) for role definitions. if try_parse_sts_request(query.as_deref()).is_some() { let config = load_sts_config(&env)?; - if let Some((status, xml)) = - try_handle_sts(query.as_deref(), &config, &jwks_cache).await - { + if let Some((status, xml)) = try_handle_sts(query.as_deref(), &config, &jwks_cache).await { return Ok(Response::builder() .status(status) .header("content-type", "application/xml") @@ -118,10 +118,16 @@ async fn fetch( let resolver = SourceCoopResolver::new(api_client); let handler = ProxyHandler::new(WorkerBackend, resolver); - return Ok( - handle_action(method, &handler, &reqwest_client, &path, query.as_deref(), &headers, body) - .await, - ); + return Ok(handle_action( + method, + &handler, + &reqwest_client, + &path, + query.as_deref(), + &headers, + body, + ) + .await); } let config = load_static_config(&env)?; @@ -129,10 +135,16 @@ async fn fetch( let resolver = DefaultResolver::new(config, virtual_host_domain); let handler = ProxyHandler::new(WorkerBackend, resolver); - Ok( - handle_action(method, &handler, &reqwest_client, &path, query.as_deref(), &headers, body) - .await, + Ok(handle_action( + method, + &handler, + &reqwest_client, + &path, + query.as_deref(), + &headers, + body, ) + .await) } // ── Two-phase request handling ────────────────────────────────────── @@ -170,11 +182,7 @@ async fn handle_action( /// /// On WASM, reqwest wraps `web_sys::fetch` internally. Bodies are collected /// to bytes since WASM reqwest doesn't support streaming. -async fn forward_to_backend( - client: &reqwest::Client, - fwd: ForwardRequest, - body: Body, -) -> Response { +async fn forward_to_backend(client: &reqwest::Client, fwd: ForwardRequest, body: Body) -> Response { let mut req_builder = client.request(fwd.method.clone(), fwd.url.as_str()); for (k, v) in fwd.headers.iter() { @@ -236,7 +244,11 @@ async fn forward_to_backend( fn load_config_from_env(env: &Env, var_name: &str) -> Result { if let Ok(var) = env.var(var_name) { let config_str = var.to_string(); - tracing::debug!(var = var_name, config_len = config_str.len(), "loaded config as string"); + tracing::debug!( + var = var_name, + config_len = config_str.len(), + "loaded config as string" + ); StaticProvider::from_json(&config_str) .map_err(|e| worker::Error::RustError(format!("{} config error: {}", var_name, e))) } else { @@ -254,8 +266,7 @@ fn load_static_config(env: &Env) -> Result { /// Load STS config: tries STS_CONFIG first, falls back to PROXY_CONFIG. fn load_sts_config(env: &Env) -> Result { - load_config_from_env(env, "STS_CONFIG") - .or_else(|_| load_config_from_env(env, "PROXY_CONFIG")) + load_config_from_env(env, "STS_CONFIG").or_else(|_| load_config_from_env(env, "PROXY_CONFIG")) } /// Load cache TTL overrides from environment variables. diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index c99df07..f6a834b 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -10,7 +10,9 @@ use http::HeaderMap; use http_body_util::BodyStream; use source_coop_core::axum::{build_proxy_response, error_response}; use source_coop_core::config::ConfigProvider; -use source_coop_core::proxy::{ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST}; +use source_coop_core::proxy::{ + ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, +}; use source_coop_core::resolver::DefaultResolver; use source_coop_sts::{try_handle_sts, JwksCache}; use std::net::SocketAddr; @@ -128,9 +130,7 @@ async fn request_handler( match action { HandlerAction::Response(result) => build_proxy_response(result), - HandlerAction::Forward(fwd) => { - forward_to_backend(&state.reqwest_client, fwd, body).await - } + HandlerAction::Forward(fwd) => forward_to_backend(&state.reqwest_client, fwd, body).await, HandlerAction::NeedsBody(pending) => { let collected = match axum::body::to_bytes(body, usize::MAX).await { Ok(b) => b, @@ -146,11 +146,7 @@ async fn request_handler( } /// Execute a Forward request via reqwest, streaming both request and response bodies. -async fn forward_to_backend( - client: &reqwest::Client, - fwd: ForwardRequest, - body: Body, -) -> Response { +async fn forward_to_backend(client: &reqwest::Client, fwd: ForwardRequest, body: Body) -> Response { let mut req_builder = client.request(fwd.method.clone(), fwd.url.as_str()); for (k, v) in fwd.headers.iter() { @@ -159,8 +155,8 @@ async fn forward_to_backend( // Attach streaming body for PUT if fwd.method == http::Method::PUT { - let body_stream = BodyStream::new(body) - .try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); + let body_stream = + BodyStream::new(body).try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); req_builder = req_builder.body(reqwest::Body::wrap_stream(body_stream)); } From 56a4a025234abf1ea03e5bf5141e81bcd2a89bfe Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 25 Feb 2026 23:40:20 -0500 Subject: [PATCH 46/82] ci: add rust caching --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1cf2721..f4b4471 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,7 @@ jobs: with: toolchain: stable components: clippy + - uses: Swatinem/rust-cache@v2 - run: cargo clippy -- -D warnings check: @@ -35,6 +36,7 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 - name: Check workspace run: cargo check - name: Check cf-workers (wasm32) @@ -48,6 +50,7 @@ jobs: - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable + - uses: Swatinem/rust-cache@v2 - run: cargo test build: @@ -58,6 +61,7 @@ jobs: - uses: dtolnay/rust-toolchain@v1 with: toolchain: stable + - uses: Swatinem/rust-cache@v2 - name: Build server run: cargo build -p source-coop-server - name: Build CLI From 553e8e3ad80bfdf7ebae36646f40b552cad196b1 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 10:22:29 -0500 Subject: [PATCH 47/82] feat(cli): persist credentials --- Cargo.lock | 40 ++++++++++ crates/cli/Cargo.toml | 2 + crates/cli/README.md | 60 ++++++++++++--- crates/cli/src/cache.rs | 159 ++++++++++++++++++++++++++++++++++++++++ crates/cli/src/main.rs | 39 +++++++++- crates/cli/src/sts.rs | 4 +- 6 files changed, 289 insertions(+), 15 deletions(-) create mode 100644 crates/cli/src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 79b31a8..6a1f740 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -712,6 +712,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1737,6 +1758,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "outref" version = "0.5.2" @@ -2062,6 +2089,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -2552,7 +2590,9 @@ name = "source-coop-cli" version = "1.0.4" dependencies = [ "base64", + "chrono", "clap", + "dirs", "open", "quick-xml 0.37.5", "rand 0.8.5", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 0f1b724..ed6f81e 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -21,6 +21,8 @@ sha2 = { workspace = true } base64 = { workspace = true } rand = "0.8" quick-xml = { workspace = true } +chrono = { workspace = true } +dirs = "6" [features] default = ["production"] diff --git a/crates/cli/README.md b/crates/cli/README.md index 0508527..13e6f3c 100644 --- a/crates/cli/README.md +++ b/crates/cli/README.md @@ -12,20 +12,57 @@ cargo install --path crates/cli ## Usage +### Recommended: login + credential-process + +1. Log in once (opens browser, caches credentials to `~/.source-coop/credentials/`): + ```bash -source-coop login --role-arn +source-coop login ``` -This opens your browser to the Source Cooperative login page. After authenticating, temporary S3 credentials are printed to stdout. +2. Configure `~/.aws/config` to use cached credentials: -### Options +```ini +[profile source-coop] +credential_process = source-coop credential-process +endpoint_url = https://data.source.coop +``` + +3. Use AWS tools normally: + +```bash +aws s3 ls s3://my-bucket/ --profile source-coop +``` + +When credentials expire, run `source-coop login` again. + +### Multiple roles + +Each role's credentials are cached separately: + +```bash +source-coop login --role-arn reader-role +source-coop login --role-arn admin-role +``` + +```ini +[profile source-coop] +credential_process = source-coop credential-process --role-arn reader-role +endpoint_url = https://data.source.coop + +[profile source-coop-admin] +credential_process = source-coop credential-process --role-arn admin-role +endpoint_url = https://data.source.coop +``` + +### Login options | Flag | Env var | Default | Description | |------|---------|---------|-------------| | `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | | `--client-id` | `SOURCE_OIDC_CLIENT_ID` | `d037d00b-...` | OAuth2 client ID | -| `--proxy-url` | `SOURCE_PROXY_URL` | `http://localhost:8787` | S3 proxy URL for STS | -| `--role-arn` | `SOURCE_ROLE_ARN` | *(required)* | Role ARN to assume | +| `--proxy-url` | `SOURCE_PROXY_URL` | `https://data.source.coop` | S3 proxy URL for STS | +| `--role-arn` | `SOURCE_ROLE_ARN` | `source-coop-user` | Role ARN to assume | | `--format` | | `credential-process` | Output format: `credential-process` or `env` | | `--duration` | | | Session duration in seconds | | `--scope` | | `openid` | OAuth2 scopes | @@ -33,17 +70,18 @@ This opens your browser to the Source Cooperative login page. After authenticati ### Output formats -**credential-process** (default) — for use with `~/.aws/config`: +In addition to caching, `login` prints credentials to stdout: -```ini -[profile source-coop] -credential_process = source-coop login --role-arn +**credential-process** (default) — AWS credential_process JSON: + +```bash +source-coop login ``` -**env** — for shell eval: +**env** — shell export statements: ```bash -eval $(source-coop login --role-arn --format env) +eval $(source-coop login --format env) ``` ## OIDC provider setup diff --git a/crates/cli/src/cache.rs b/crates/cli/src/cache.rs new file mode 100644 index 0000000..d70f18d --- /dev/null +++ b/crates/cli/src/cache.rs @@ -0,0 +1,159 @@ +use crate::sts::Credentials; +use chrono::Utc; +use std::fs; +use std::io; +use std::path::PathBuf; + +/// Replace any character that isn't alphanumeric, `-`, or `_` with `_`. +fn sanitize_role_arn(role_arn: &str) -> String { + role_arn + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +/// Full path to the credentials cache file for a given role. +fn cache_path(role_arn: &str) -> Result { + let home = dirs::home_dir().ok_or("Could not determine home directory")?; + let sanitized = sanitize_role_arn(role_arn); + Ok(home + .join(".source-coop") + .join("credentials") + .join(format!("{sanitized}.json"))) +} + +/// Write credentials to the per-role cache file. +/// Creates `~/.source-coop/credentials/` if it does not exist. +/// Sets file permissions to 0600 on Unix. +pub fn write_credentials(role_arn: &str, creds: &Credentials) -> Result { + let path = cache_path(role_arn)?; + let dir = path.parent().unwrap(); + + fs::create_dir_all(dir) + .map_err(|e| format!("Failed to create cache directory {}: {e}", dir.display()))?; + + let json = serde_json::to_string_pretty(creds) + .map_err(|e| format!("Failed to serialize credentials: {e}"))?; + + fs::write(&path, &json) + .map_err(|e| format!("Failed to write credentials cache {}: {e}", path.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)) + .map_err(|e| format!("Failed to set permissions on {}: {e}", path.display()))?; + } + + Ok(path) +} + +/// Read credentials from the cache file for a given role. +/// Returns `None` if the file does not exist. +pub fn read_credentials(role_arn: &str) -> Result, String> { + let path = cache_path(role_arn)?; + match fs::read_to_string(&path) { + Ok(contents) => { + let creds: Credentials = serde_json::from_str(&contents) + .map_err(|e| format!("Failed to parse credentials cache: {e}"))?; + Ok(Some(creds)) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(format!( + "Failed to read credentials cache {}: {e}", + path.display() + )), + } +} + +/// Check if credentials are expired or will expire within a 60-second buffer. +pub fn is_expired(creds: &Credentials) -> Result { + let expiration = chrono::DateTime::parse_from_rfc3339(&creds.expiration).map_err(|e| { + format!( + "Failed to parse expiration timestamp '{}': {e}", + creds.expiration + ) + })?; + + let now = Utc::now(); + let buffer = chrono::Duration::seconds(60); + + Ok(expiration <= now + buffer) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_creds(expiration: &str) -> Credentials { + Credentials { + access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(), + session_token: "FwoGZXIvYXdzEtest".to_string(), + expiration: expiration.to_string(), + } + } + + #[test] + fn sanitize_simple_name() { + assert_eq!(sanitize_role_arn("source-coop-user"), "source-coop-user"); + } + + #[test] + fn sanitize_arn_with_special_chars() { + assert_eq!( + sanitize_role_arn("arn:aws:iam::123:role/Foo"), + "arn_aws_iam__123_role_Foo" + ); + } + + #[test] + fn sanitize_preserves_underscores() { + assert_eq!(sanitize_role_arn("my_role-name"), "my_role-name"); + } + + #[test] + fn expired_future_date() { + let future = (Utc::now() + chrono::Duration::hours(1)).to_rfc3339(); + let creds = sample_creds(&future); + assert!(!is_expired(&creds).unwrap()); + } + + #[test] + fn expired_past_date() { + let past = (Utc::now() - chrono::Duration::hours(1)).to_rfc3339(); + let creds = sample_creds(&past); + assert!(is_expired(&creds).unwrap()); + } + + #[test] + fn expired_within_buffer() { + // 30 seconds from now is within the 60s buffer + let near_future = (Utc::now() + chrono::Duration::seconds(30)).to_rfc3339(); + let creds = sample_creds(&near_future); + assert!(is_expired(&creds).unwrap()); + } + + #[test] + fn expired_invalid_timestamp() { + let creds = sample_creds("not-a-timestamp"); + assert!(is_expired(&creds).is_err()); + } + + #[test] + fn round_trip_serialization() { + let creds = sample_creds("2026-03-01T00:00:00Z"); + let json = serde_json::to_string_pretty(&creds).unwrap(); + let loaded: Credentials = serde_json::from_str(&json).unwrap(); + assert_eq!(loaded.access_key_id, creds.access_key_id); + assert_eq!(loaded.secret_access_key, creds.secret_access_key); + assert_eq!(loaded.session_token, creds.session_token); + assert_eq!(loaded.expiration, creds.expiration); + } +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 4e28679..d5ef051 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,3 +1,4 @@ +mod cache; mod oidc; mod output; mod sts; @@ -31,6 +32,8 @@ struct Cli { enum Commands { /// Authenticate via OIDC and obtain temporary S3 credentials Login(LoginArgs), + /// Output cached credentials for AWS credential_process + CredentialProcess(CredentialProcessArgs), } #[derive(Parser)] @@ -68,6 +71,13 @@ struct LoginArgs { port: u16, } +#[derive(Parser)] +struct CredentialProcessArgs { + /// Role ARN to read cached credentials for + #[arg(long, env = "SOURCE_ROLE_ARN", default_value = defaults::ROLE_ARN)] + role_arn: String, +} + #[derive(Clone, ValueEnum)] enum OutputFormat { /// AWS credential_process JSON format @@ -87,6 +97,12 @@ async fn main() { std::process::exit(1); } } + Commands::CredentialProcess(args) => { + if let Err(e) = run_credential_process(args) { + eprintln!("Error: {e}"); + std::process::exit(1); + } + } } } @@ -101,9 +117,14 @@ async fn run_login(args: LoginArgs) -> Result<(), String> { // 3. STS credential exchange eprintln!("Exchanging token for credentials..."); - let creds = sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; + let creds = + sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; + + // 4. Cache credentials + let path = cache::write_credentials(&args.role_arn, &creds)?; + eprintln!("Credentials cached to {}", path.display()); - // 4. Output + // 5. Output match args.format { OutputFormat::CredentialProcess => output::print_credential_process(&creds), OutputFormat::Env => output::print_env(&creds), @@ -111,3 +132,17 @@ async fn run_login(args: LoginArgs) -> Result<(), String> { Ok(()) } + +fn run_credential_process(args: CredentialProcessArgs) -> Result<(), String> { + let creds = cache::read_credentials(&args.role_arn)? + .ok_or("No cached credentials found. Run 'source-coop login' first.")?; + + if cache::is_expired(&creds)? { + return Err( + "Cached credentials have expired. Run 'source-coop login' to refresh.".to_string(), + ); + } + + output::print_credential_process(&creds); + Ok(()) +} diff --git a/crates/cli/src/sts.rs b/crates/cli/src/sts.rs index 391ebcd..58a5306 100644 --- a/crates/cli/src/sts.rs +++ b/crates/cli/src/sts.rs @@ -1,7 +1,7 @@ use quick_xml::de::from_str as xml_from_str; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Credentials { pub access_key_id: String, pub secret_access_key: String, From f9bf91082a23100aa663f6bf86ad6a9737da718e Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 10:36:51 -0500 Subject: [PATCH 48/82] fix(core): support public data connections --- crates/libs/api/src/api.rs | 3 ++- crates/libs/api/src/resolver.rs | 30 +++++++++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/libs/api/src/api.rs b/crates/libs/api/src/api.rs index f2c5330..5757cff 100644 --- a/crates/libs/api/src/api.rs +++ b/crates/libs/api/src/api.rs @@ -85,7 +85,8 @@ pub struct ProductMirror { #[derive(Debug, Deserialize)] pub struct DataConnection { pub details: ConnectionDetails, - pub authentication: ConnectionAuth, + #[serde(default)] + pub authentication: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/libs/api/src/resolver.rs b/crates/libs/api/src/resolver.rs index ae4f5d4..d8a43ed 100644 --- a/crates/libs/api/src/resolver.rs +++ b/crates/libs/api/src/resolver.rs @@ -97,13 +97,17 @@ impl SourceCoopResolver { opts.insert("endpoint".to_string(), endpoint); opts.insert("bucket_name".to_string(), bucket); opts.insert("region".to_string(), region); - if let Some(ak) = &conn.authentication.access_key_id { - opts.insert("access_key_id".to_string(), ak.clone()); - } - if let Some(sk) = &conn.authentication.secret_access_key { - opts.insert("secret_access_key".to_string(), sk.clone()); - } - if conn.authentication.access_key_id.is_none() { + if let Some(ref auth) = conn.authentication { + if let Some(ak) = &auth.access_key_id { + opts.insert("access_key_id".to_string(), ak.clone()); + } + if let Some(sk) = &auth.secret_access_key { + opts.insert("secret_access_key".to_string(), sk.clone()); + } + if auth.access_key_id.is_none() { + opts.insert("skip_signature".to_string(), "true".to_string()); + } + } else { opts.insert("skip_signature".to_string(), "true".to_string()); } opts @@ -116,10 +120,14 @@ impl SourceCoopResolver { if let Some(container) = &conn.details.container_name { opts.insert("container_name".to_string(), container.clone()); } - if let Some(key) = &conn.authentication.access_key { - opts.insert("access_key".to_string(), key.clone()); - } - if conn.authentication.access_key.is_none() { + if let Some(ref auth) = conn.authentication { + if let Some(key) = &auth.access_key { + opts.insert("access_key".to_string(), key.clone()); + } + if auth.access_key.is_none() { + opts.insert("skip_signature".to_string(), "true".to_string()); + } + } else { opts.insert("skip_signature".to_string(), "true".to_string()); } opts From 1b6207388a1f7b07b760fd35b849e3ce08915a3b Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 10:39:28 -0500 Subject: [PATCH 49/82] fix(api): correctly parse truthy values from config --- crates/libs/api/src/api.rs | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/crates/libs/api/src/api.rs b/crates/libs/api/src/api.rs index 5757cff..4f1fa69 100644 --- a/crates/libs/api/src/api.rs +++ b/crates/libs/api/src/api.rs @@ -127,14 +127,45 @@ pub enum RepositoryPermission { } /// API response for the permissions endpoint. +/// +/// The API may return permission fields as booleans (`true`/`false`) or as +/// strings (e.g., `"read"`, `"write"`). The custom deserializer handles both. #[derive(Debug, Deserialize)] pub struct PermissionsResponse { - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_truthy")] pub read: bool, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_truthy")] pub write: bool, } +/// Deserialize a value as `true` if it is a boolean `true` or any non-empty string. +fn deserialize_truthy<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + use serde::de; + + struct TruthyVisitor; + + impl<'de> de::Visitor<'de> for TruthyVisitor { + type Value = bool; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a boolean or a string") + } + + fn visit_bool(self, v: bool) -> Result { + Ok(v) + } + + fn visit_str(self, v: &str) -> Result { + Ok(!v.is_empty()) + } + } + + deserializer.deserialize_any(TruthyVisitor) +} + /// API response for listing products under an account. #[derive(Debug, Deserialize)] pub struct AccountResponse { From f6ce6dbccc68d4bcee79dc367e18bbeec572d1df Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 10:40:55 -0500 Subject: [PATCH 50/82] chore(cf-workers): ignore deadcode warning --- crates/runtimes/cf-workers/src/tracing_layer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/runtimes/cf-workers/src/tracing_layer.rs b/crates/runtimes/cf-workers/src/tracing_layer.rs index 8aa8332..2c52590 100644 --- a/crates/runtimes/cf-workers/src/tracing_layer.rs +++ b/crates/runtimes/cf-workers/src/tracing_layer.rs @@ -28,6 +28,7 @@ impl WorkerSubscriber { } } + #[allow(dead_code)] pub fn with_max_level(mut self, level: Level) -> Self { self.max_level = level; self From a5c0f965512128213e38a3a4438ed1c7718a59f9 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 11:44:03 -0500 Subject: [PATCH 51/82] refator(core): better handle various upstream errors --- crates/libs/api/src/api.rs | 31 +++++++++++++++++++----- crates/libs/api/src/resolver.rs | 29 ++++++++++++---------- crates/libs/core/src/error.rs | 12 ++++----- crates/runtimes/cf-workers/src/client.rs | 9 +++++-- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/crates/libs/api/src/api.rs b/crates/libs/api/src/api.rs index 4f1fa69..8af4143 100644 --- a/crates/libs/api/src/api.rs +++ b/crates/libs/api/src/api.rs @@ -18,13 +18,16 @@ pub struct CacheOptions { } /// Trait abstracting HTTP JSON fetching so each runtime can provide its own implementation. +/// +/// `fetch_json` returns `Ok(Some(T))` on 2xx, `Ok(None)` on 404 (resource not +/// found), and `Err` for all other failures (network errors, 5xx, 429, etc.). pub trait HttpClient: Clone + MaybeSend + MaybeSync + 'static { fn fetch_json( &self, url: &str, headers: &[(&str, &str)], cache: Option<&CacheOptions>, - ) -> impl Future> + MaybeSend; + ) -> impl Future, ProxyError>> + MaybeSend; } /// Per-endpoint cache TTLs (seconds). Set to 0 to disable caching. @@ -199,11 +202,13 @@ impl SourceApiClient { } /// `GET /api/v1/products/{account_id}/{repo_id}` + /// + /// Returns `Ok(None)` when the API returns 404 (product does not exist). pub async fn get_product( &self, account_id: &str, repo_id: &str, - ) -> Result { + ) -> Result, ProxyError> { let url = format!( "{}/api/v1/products/{}/{}", self.api_url, account_id, repo_id @@ -218,7 +223,12 @@ impl SourceApiClient { } /// `GET /api/v1/data-connections/{id}` - pub async fn get_data_connection(&self, id: &str) -> Result { + /// + /// Returns `Ok(None)` when the API returns 404 (connection does not exist). + pub async fn get_data_connection( + &self, + id: &str, + ) -> Result, ProxyError> { let url = format!("{}/api/v1/data-connections/{}", self.api_url, id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); @@ -230,7 +240,12 @@ impl SourceApiClient { } /// `GET /api/v1/api-keys/{access_key_id}/auth` - pub async fn get_api_key(&self, access_key_id: &str) -> Result { + /// + /// Returns `Ok(None)` when the API returns 404 (key does not exist). + pub async fn get_api_key( + &self, + access_key_id: &str, + ) -> Result, ProxyError> { let url = format!("{}/api/v1/api-keys/{}/auth", self.api_url, access_key_id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); @@ -248,12 +263,14 @@ impl SourceApiClient { /// /// A custom `cacheKey` incorporating the user's API key prevents /// cross-user cache poisoning (the URL is the same for all users). + /// + /// Returns `Ok(None)` when the API returns 404. pub async fn get_permissions( &self, account_id: &str, repo_id: &str, user_api_key: &str, - ) -> Result { + ) -> Result, ProxyError> { let url = format!( "{}/api/v1/products/{}/{}/permissions", self.api_url, account_id, repo_id @@ -270,10 +287,12 @@ impl SourceApiClient { } /// `GET /api/v1/accounts/{account_id}` + /// + /// Returns `Ok(None)` when the API returns 404 (account does not exist). pub async fn list_account_repos( &self, account_id: &str, - ) -> Result { + ) -> Result, ProxyError> { let url = format!("{}/api/v1/products/{}", self.api_url, account_id); let auth = self.auth_headers(); let headers: Vec<(&str, &str)> = auth.iter().map(|(k, v)| (*k, v.as_str())).collect(); diff --git a/crates/libs/api/src/resolver.rs b/crates/libs/api/src/resolver.rs index d8a43ed..49c3840 100644 --- a/crates/libs/api/src/resolver.rs +++ b/crates/libs/api/src/resolver.rs @@ -40,16 +40,8 @@ impl SourceCoopResolver { let product = self .api_client .get_product(account_id, repo_id) - .await - .map_err(|e| { - tracing::warn!( - account_id = account_id, - repo_id = repo_id, - error = %e, - "failed to fetch product from Source API" - ); - ProxyError::BucketNotFound(bucket_name.clone()) - })?; + .await? + .ok_or_else(|| ProxyError::BucketNotFound(bucket_name.clone()))?; if product.disabled { return Err(ProxyError::BucketNotFound(bucket_name)); @@ -69,7 +61,13 @@ impl SourceCoopResolver { let conn = self .api_client .get_data_connection(&mirror.connection_id) - .await?; + .await? + .ok_or_else(|| { + ProxyError::ConfigError(format!( + "data connection '{}' not found", + mirror.connection_id + )) + })?; let base_prefix = conn.details.base_prefix.unwrap_or_default(); @@ -173,7 +171,8 @@ impl SourceCoopResolver { let perms = self .api_client .get_permissions(account_id, repo_id, &sig.access_key_id) - .await?; + .await? + .ok_or(ProxyError::AccessDenied)?; let is_write = matches!(*method, Method::PUT | Method::POST | Method::DELETE); @@ -206,7 +205,11 @@ impl SourceCoopResolver { account_id = account_id, "handling account listing for account" ); - let account = self.api_client.list_account_repos(account_id).await?; + let account = self + .api_client + .list_account_repos(account_id) + .await? + .ok_or_else(|| ProxyError::BucketNotFound(account_id.to_string()))?; tracing::info!( account_id = account_id, diff --git a/crates/libs/core/src/error.rs b/crates/libs/core/src/error.rs index cfd1a01..816c13c 100644 --- a/crates/libs/core/src/error.rs +++ b/crates/libs/core/src/error.rs @@ -60,7 +60,7 @@ impl ProxyError { Self::ExpiredCredentials => "ExpiredToken", Self::InvalidOidcToken(_) => "InvalidIdentityToken", Self::RoleNotFound(_) => "AccessDenied", - Self::BackendError(_) => "InternalError", + Self::BackendError(_) => "ServiceUnavailable", Self::PreconditionFailed => "PreconditionFailed", Self::NotModified => "NotModified", Self::ConfigError(_) => "InternalError", @@ -79,21 +79,21 @@ impl ProxyError { Self::RoleNotFound(_) => 403, Self::PreconditionFailed => 412, Self::NotModified => 304, - Self::BackendError(_) | Self::ConfigError(_) | Self::Internal(_) => 500, + Self::BackendError(_) => 503, + Self::ConfigError(_) | Self::Internal(_) => 500, } } /// Return a message safe to show to external clients. /// - /// For server-side errors (500), returns a generic message to avoid + /// For server-side errors (5xx), returns a generic message to avoid /// leaking backend infrastructure details. For client errors (4xx), /// returns the full message (the client already knows the bucket name, /// key, etc.). pub fn safe_message(&self) -> String { match self { - Self::BackendError(_) | Self::ConfigError(_) | Self::Internal(_) => { - "Internal server error".to_string() - } + Self::BackendError(_) => "Service unavailable".to_string(), + Self::ConfigError(_) | Self::Internal(_) => "Internal server error".to_string(), other => other.to_string(), } } diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index ef57d63..215d2cc 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -42,7 +42,7 @@ impl HttpClient for WorkerHttpClient { url: &str, headers: &[(&str, &str)], cache: Option<&CacheOptions>, - ) -> Result { + ) -> Result, ProxyError> { // Check cache for a hit before making the request. let cache_state = if let Some(opts) = cache { let key = cache_key_url(url, opts); @@ -51,7 +51,7 @@ impl HttpClient for WorkerHttpClient { Ok(Some(mut cached)) => { if let Ok(text) = cached.text().await { if let Ok(value) = serde_json::from_str(&text) { - return Ok(value); + return Ok(Some(value)); } } // Cache hit but couldn't deserialize — fall through to fetch. @@ -88,6 +88,10 @@ impl HttpClient for WorkerHttpClient { .await .map_err(|e| ProxyError::Internal(format!("failed to read text: {}", e)))?; + if status == 404 { + return Ok(None); + } + if !(200..300).contains(&status) { return Err(ProxyError::BackendError(format!( "API request to {} returned status {}", @@ -108,6 +112,7 @@ impl HttpClient for WorkerHttpClient { } serde_json::from_str(&text) + .map(Some) .map_err(|e| ProxyError::Internal(format!("failed to deserialize response: {}", e))) } } From 3b78af57157321cde30a230a09eaf44a32d47002 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 26 Feb 2026 23:44:10 -0500 Subject: [PATCH 52/82] feat: use "sealed tokens" for sessions --- .gitignore | 2 +- Cargo.lock | 103 ++++ Cargo.toml | 1 + README.md | 10 +- crates/libs/core/Cargo.toml | 1 + crates/libs/core/README.md | 17 +- crates/libs/core/src/auth.rs | 525 +++++++++++++----- crates/libs/core/src/config/cached.rs | 22 +- crates/libs/core/src/config/dynamodb.rs | 63 +-- crates/libs/core/src/config/http.rs | 45 +- crates/libs/core/src/config/mod.rs | 17 +- crates/libs/core/src/config/postgres.rs | 51 +- crates/libs/core/src/config/static_file.rs | 55 +- crates/libs/core/src/lib.rs | 1 + crates/libs/core/src/resolver.rs | 21 +- crates/libs/core/src/sealed_token.rs | 165 ++++++ crates/libs/sts/src/lib.rs | 26 +- crates/libs/sts/src/sts.rs | 6 +- crates/runtimes/cf-workers/src/lib.rs | 20 +- crates/runtimes/cf-workers/wrangler.toml | 7 +- .../server/src/bin/source-coop-proxy.rs | 7 + crates/runtimes/server/src/server.rs | 12 +- 22 files changed, 793 insertions(+), 384 deletions(-) create mode 100644 crates/libs/core/src/sealed_token.rs diff --git a/.gitignore b/.gitignore index 95fe15f..a55e53a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ scripts/task_definition.json target .wrangler -crates/runtimes/cf-workers/.env +.env* diff --git a/Cargo.lock b/Cargo.lock index 6a1f740..c7433cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,41 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -535,6 +570,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.60" @@ -677,9 +722,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "der" version = "0.7.10" @@ -1003,6 +1058,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "h2" version = "0.3.27" @@ -1426,6 +1491,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1741,6 +1815,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.3.3" @@ -1879,6 +1959,18 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -2608,6 +2700,7 @@ dependencies = [ name = "source-coop-core" version = "1.0.4" dependencies = [ + "aes-gcm", "async-trait", "aws-sdk-dynamodb", "axum", @@ -3317,6 +3410,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 2e7137b..72df8a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ url = "2" hmac = "0.12" sha2 = { version = "0.10", features = ["oid"] } rsa = "0.9" +aes-gcm = "0.10" # Object store object_store = { version = "0.12", default-features = false, features = ["aws"] } diff --git a/README.md b/README.md index 5ab3176..174fe2d 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,6 @@ impl ConfigProvider for MyProvider { async fn get_bucket(&self, name: &str) -> Result, ProxyError> { todo!() } async fn get_role(&self, role_id: &str) -> Result, ProxyError> { todo!() } async fn get_credential(&self, access_key_id: &str) -> Result, ProxyError> { todo!() } - async fn store_temporary_credential(&self, cred: &TemporaryCredentials) -> Result<(), ProxyError> { todo!() } - async fn get_temporary_credential(&self, access_key_id: &str) -> Result, ProxyError> { todo!() } } ``` @@ -276,7 +274,7 @@ provider.invalidate_all(); provider.invalidate_bucket("my-bucket"); ``` -The cache is thread-safe (`RwLock`-based) and evicts entries lazily on access. Temporary credential writes and reads always go directly to the underlying provider — they're already short-lived and caching them would create security issues with stale session tokens. +The cache is thread-safe (`RwLock`-based) and evicts entries lazily on access. ### Roles @@ -369,6 +367,12 @@ jobs: The proxy validates the JWT against the OIDC provider's JWKS, checks the trust policy (issuer, audience, subject conditions with glob matching), and mints temporary credentials scoped to the role's allowed buckets/prefixes. +**Sealed session tokens:** When `SESSION_TOKEN_KEY` is configured (a base64-encoded 32-byte AES-256-GCM key), the full `TemporaryCredentials` are encrypted into the session token itself. The proxy decrypts the token on each subsequent request — no server-side credential storage is needed. This is required for stateless runtimes like Cloudflare Workers. Generate a key with: + +```bash +openssl rand -base64 32 +``` + ## Multi-Runtime Design The crate workspace separates concerns so the core logic compiles to both native and WASM targets: diff --git a/crates/libs/core/Cargo.toml b/crates/libs/core/Cargo.toml index 75e7db8..adfe7d3 100644 --- a/crates/libs/core/Cargo.toml +++ b/crates/libs/core/Cargo.toml @@ -29,6 +29,7 @@ hex.workspace = true url.workspace = true hmac.workspace = true sha2.workspace = true +aes-gcm.workspace = true quick-xml.workspace = true tracing.workspace = true object_store.workspace = true diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md index bb59c2e..ebe0ac4 100644 --- a/crates/libs/core/README.md +++ b/crates/libs/core/README.md @@ -43,6 +43,7 @@ src/ ├── error.rs ProxyError with S3-compatible error codes ├── proxy.rs ProxyHandler — the main request handler ├── resolver.rs RequestResolver trait, ResolvedAction, DefaultResolver +├── sealed_token.rs AES-256-GCM encrypted session tokens (TokenKey) ├── s3/ │ ├── request.rs Parse incoming HTTP → S3Operation enum │ ├── response.rs Serialize S3 XML responses @@ -66,7 +67,11 @@ use source_coop_core::config::static_file::StaticProvider; let backend = MyBackend::new(); let config = StaticProvider::from_file("config.toml")?; -let resolver = DefaultResolver::new(config, Some("s3.example.com".into())); +// Optional: enable sealed session tokens for STS temporary credentials. +// When set, TemporaryCredentials are AES-256-GCM encrypted into the session +// token itself — no server-side storage needed (critical for stateless runtimes). +let token_key = None; // or Some(TokenKey::from_base64(&key_b64)?) +let resolver = DefaultResolver::new(config, Some("s3.example.com".into()), token_key); let handler = ProxyHandler::new(backend, resolver); @@ -106,6 +111,16 @@ let handler = ProxyHandler::new(backend, MyResolver::new()); See `crates/libs/source-coop/src/resolver.rs` for a real-world example that maps a `/{account}/{repo}/{key}` namespace to dynamically-resolved S3 backends with external API authorization. +## Sealed Session Tokens + +The `sealed_token` module provides stateless temporary credential verification using AES-256-GCM. When a `TokenKey` is configured (via `SESSION_TOKEN_KEY`), the STS handler encrypts the full `TemporaryCredentials` struct into the session token itself. On subsequent requests, `resolve_identity()` decrypts the token to recover the credentials — no server-side storage or config lookup is needed. + +This is critical for stateless runtimes like Cloudflare Workers where in-memory state does not persist across invocations. The `TokenKey` wraps `Arc` and is `Clone + Send + Sync`. + +Token format: `base64url(nonce[12] || ciphertext + tag)`. Expired tokens return `Err(ExpiredCredentials)`. Tokens that fail decryption (wrong key, not a sealed token) return `Ok(None)` allowing graceful rejection. + +Note: because scopes are sealed into the token at mint time, changes to a role's `allowed_scopes` in config only take effect for newly minted credentials — existing tokens retain the scopes they were issued with. + ## Feature Flags All optional — the default build has zero network dependencies: diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 7751568..c75770f 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -7,6 +7,7 @@ use crate::config::ConfigProvider; use crate::error::ProxyError; +use crate::sealed_token::TokenKey; use crate::types::{Action, ResolvedIdentity, S3Operation}; use hmac::{Hmac, Mac}; use http::HeaderMap; @@ -99,9 +100,14 @@ pub fn verify_sigv4_signature( let signed_headers_str = auth.signed_headers.join(";"); + // SigV4 requires query parameters sorted alphabetically by key (then value). + // The raw query string from the URL may not be sorted, but the client SDK + // sorts them when constructing the canonical request for signing. + let canonical_query = canonicalize_query_string(query_string); + let canonical_request = format!( "{}\n{}\n{}\n{}\n{}\n{}", - method, uri_path, query_string, canonical_headers, signed_headers_str, payload_hash + method, uri_path, canonical_query, canonical_headers, signed_headers_str, payload_hash ); let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); @@ -132,11 +138,29 @@ pub fn verify_sigv4_signature( let expected_signature = hex::encode(hmac_sha256(&signing_key, string_to_sign.as_bytes())?); - // Constant-time comparison - Ok(constant_time_eq( - expected_signature.as_bytes(), - auth.signature.as_bytes(), - )) + let matched = constant_time_eq(expected_signature.as_bytes(), auth.signature.as_bytes()); + + if !matched { + tracing::warn!( + canonical_request = %canonical_request, + string_to_sign = %string_to_sign, + expected_signature = %expected_signature, + provided_signature = %auth.signature, + "SigV4 signature mismatch — compare canonical_request with client-side (aws --debug)" + ); + } + + Ok(matched) +} + +/// Sort query string parameters for SigV4 canonical request construction. +fn canonicalize_query_string(query: &str) -> String { + if query.is_empty() { + return String::new(); + } + let mut parts: Vec<&str> = query.split('&').collect(); + parts.sort_unstable(); + parts.join("&") } fn hmac_sha256(key: &[u8], data: &[u8]) -> Result, ProxyError> { @@ -166,6 +190,7 @@ pub async fn resolve_identity( query_string: &str, headers: &HeaderMap, config: &C, + token_key: Option<&TokenKey>, ) -> Result { let auth_header = match headers.get("authorization").and_then(|v| v.to_str().ok()) { Some(h) => h, @@ -183,36 +208,55 @@ pub async fn resolve_identity( .and_then(|v| v.to_str().ok()) .unwrap_or("UNSIGNED-PAYLOAD"); - // Check for temporary credentials first (session token present) - if headers.get("x-amz-security-token").is_some() { - if let Some(temp_cred) = config.get_temporary_credential(&sig.access_key_id).await? { - // Verify session token matches (constant-time to avoid timing leaks) - let session_token = headers - .get("x-amz-security-token") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - if !constant_time_eq(session_token.as_bytes(), temp_cred.session_token.as_bytes()) { - return Err(ProxyError::AccessDenied); + // Temporary credentials: decrypt the session token to recover credentials + if let Some(session_token) = headers + .get("x-amz-security-token") + .and_then(|v| v.to_str().ok()) + { + let key = token_key.ok_or_else(|| { + tracing::warn!("session token present but no token_key configured"); + ProxyError::AccessDenied + })?; + + match key.unseal(session_token)? { + Some(creds) => { + if !constant_time_eq( + sig.access_key_id.as_bytes(), + creds.access_key_id.as_bytes(), + ) { + tracing::warn!( + header_key = %sig.access_key_id, + sealed_key = %creds.access_key_id, + "access key mismatch between auth header and sealed token" + ); + return Err(ProxyError::AccessDenied); + } + if !verify_sigv4_signature( + method, + uri_path, + query_string, + headers, + &sig, + &creds.secret_access_key, + payload_hash, + )? { + return Err(ProxyError::SignatureDoesNotMatch); + } + tracing::debug!( + access_key = %creds.access_key_id, + role = %creds.assumed_role_id, + scopes = ?creds.allowed_scopes, + "sealed token identity resolved" + ); + return Ok(ResolvedIdentity::Temporary { + credentials: creds, + }); } - - // Verify SigV4 signature - if !verify_sigv4_signature( - method, - uri_path, - query_string, - headers, - &sig, - &temp_cred.secret_access_key, - payload_hash, - )? { - return Err(ProxyError::SignatureDoesNotMatch); + None => { + tracing::warn!("session token could not be unsealed (decryption failed)"); + return Err(ProxyError::AccessDenied); } - - return Ok(ResolvedIdentity::Temporary { - credentials: temp_cred, - }); } - return Err(ProxyError::ExpiredCredentials); } // Check long-lived credentials @@ -257,7 +301,6 @@ mod tests { #[derive(Clone)] struct MockConfig { credentials: Vec, - temp_credentials: Vec, } impl MockConfig { @@ -276,33 +319,12 @@ mod tests { expires_at: None, enabled: true, }], - temp_credentials: vec![], - } - } - - fn with_temp_credential(secret: &str, session_token: &str) -> Self { - Self { - credentials: vec![], - temp_credentials: vec![TemporaryCredentials { - access_key_id: "ASIATEMP1234EXAMPLE".into(), - secret_access_key: secret.into(), - session_token: session_token.into(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - allowed_scopes: vec![AccessScope { - bucket: "test-bucket".into(), - prefixes: vec![], - actions: vec![Action::GetObject], - }], - assumed_role_id: "role-1".into(), - source_identity: "test".into(), - }], } } fn empty() -> Self { Self { credentials: vec![], - temp_credentials: vec![], } } } @@ -327,22 +349,6 @@ mod tests { .find(|c| c.access_key_id == access_key_id) .cloned()) } - async fn store_temporary_credential( - &self, - _: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - Ok(()) - } - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - Ok(self - .temp_credentials - .iter() - .find(|c| c.access_key_id == access_key_id) - .cloned()) - } } // ── Test signing helper ─────────────────────────────────────────── @@ -375,9 +381,12 @@ mod tests { let signed_headers_str = signed_header_names.join(";"); + // AWS SDKs sort query parameters when constructing the canonical request + let canonical_query = canonicalize_query_string(query_string); + let canonical_request = format!( "{}\n{}\n{}\n{}\n{}\n{}", - method, uri_path, query_string, canonical_headers, signed_headers_str, payload_hash + method, uri_path, canonical_query, canonical_headers, signed_headers_str, payload_hash ); let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); @@ -450,6 +459,7 @@ mod tests { "", &headers, &config, + None, ) .await .unwrap(); @@ -471,6 +481,54 @@ mod tests { "", &headers, &config, + None, + ) + .await + .unwrap(); + + assert!(matches!(identity, ResolvedIdentity::LongLived { .. })); + }); + } + + #[test] + fn valid_signature_with_unsorted_query_params() { + run(async { + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let config = MockConfig::with_credential(secret); + + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let payload_hash = "UNSIGNED-PAYLOAD"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "s3.example.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + + // Sign with sorted query (as AWS SDKs do internally) + let auth = sign_request( + &http::Method::GET, + "/test-bucket", + "list-type=2&prefix=&delimiter=%2F&encoding-type=url", + &headers, + "AKIAIOSFODNN7EXAMPLE", + secret, + date_stamp, + amz_date, + "us-east-1", + &["host", "x-amz-content-sha256", "x-amz-date"], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + + // Pass UNSORTED query string (as it arrives from the raw URL) + let identity = resolve_identity( + &http::Method::GET, + "/test-bucket", + "list-type=2&prefix=&delimiter=%2F&encoding-type=url", + &headers, + &config, + None, ) .await .unwrap(); @@ -494,6 +552,7 @@ mod tests { "", &headers, &config, + None, ) .await .unwrap_err(); @@ -531,6 +590,7 @@ mod tests { "", &headers, &config, + None, ) .await .unwrap_err(); @@ -551,6 +611,7 @@ mod tests { "", &headers, &config, + None, ) .await .unwrap_err(); @@ -560,11 +621,20 @@ mod tests { } #[test] - fn temp_credential_valid_signature_and_token() { + fn sealed_token_wrong_session_token_is_rejected() { + use crate::sealed_token::TokenKey; + run(async { + let key_bytes = [0x42u8; 32]; + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + key_bytes, + ); + let token_key = TokenKey::from_base64(&encoded).unwrap(); + let config = MockConfig::empty(); + let secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; - let config = MockConfig::with_temp_credential(secret, session_token); + let wrong_token = "NOT_A_SEALED_TOKEN_AT_ALL"; let date_stamp = "20240101"; let amz_date = "20240101T000000Z"; @@ -574,7 +644,7 @@ mod tests { headers.insert("host", "s3.example.com".parse().unwrap()); headers.insert("x-amz-date", amz_date.parse().unwrap()); headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", session_token.parse().unwrap()); + headers.insert("x-amz-security-token", wrong_token.parse().unwrap()); let auth = sign_request( &http::Method::GET, @@ -596,27 +666,52 @@ mod tests { ); headers.insert("authorization", auth.parse().unwrap()); - let identity = resolve_identity( + let err = resolve_identity( &http::Method::GET, "/test-bucket/key.txt", "", &headers, &config, + Some(&token_key), ) .await - .unwrap(); + .unwrap_err(); - assert!(matches!(identity, ResolvedIdentity::Temporary { .. })); + assert!(matches!(err, ProxyError::AccessDenied)); }); } #[test] - fn temp_credential_wrong_session_token_is_rejected() { + fn sealed_token_wrong_signature_is_rejected() { + use crate::sealed_token::TokenKey; + use crate::types::AccessScope; + run(async { - let secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let real_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; - let wrong_token = "WRONG_TOKEN_VALUE_HERE"; - let config = MockConfig::with_temp_credential(secret, real_token); + let key_bytes = [0x42u8; 32]; + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + key_bytes, + ); + let token_key = TokenKey::from_base64(&encoded).unwrap(); + + let real_secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; + let creds = TemporaryCredentials { + access_key_id: "ASIATEMP1234EXAMPLE".into(), + secret_access_key: real_secret.into(), + session_token: String::new(), + expiration: chrono::Utc::now() + chrono::Duration::hours(1), + allowed_scopes: vec![AccessScope { + bucket: "test-bucket".into(), + prefixes: vec![], + actions: vec![Action::GetObject], + }], + assumed_role_id: "role-1".into(), + source_identity: "test".into(), + }; + + let sealed = token_key.seal(&creds).unwrap(); + let config = MockConfig::empty(); let date_stamp = "20240101"; let amz_date = "20240101T000000Z"; @@ -626,15 +721,16 @@ mod tests { headers.insert("host", "s3.example.com".parse().unwrap()); headers.insert("x-amz-date", amz_date.parse().unwrap()); headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", wrong_token.parse().unwrap()); + headers.insert("x-amz-security-token", sealed.parse().unwrap()); + // Sign with wrong secret — sealed token is valid but sig won't match let auth = sign_request( &http::Method::GET, "/test-bucket/key.txt", "", &headers, "ASIATEMP1234EXAMPLE", - secret, + wrong_secret, date_stamp, amz_date, "us-east-1", @@ -654,6 +750,35 @@ mod tests { "", &headers, &config, + Some(&token_key), + ) + .await + .unwrap_err(); + + assert!( + matches!(err, ProxyError::SignatureDoesNotMatch), + "expected SignatureDoesNotMatch, got: {:?}", + err + ); + }); + } + + #[test] + fn disabled_credential_is_rejected_before_sig_check() { + run(async { + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let mut config = MockConfig::with_credential(secret); + config.credentials[0].enabled = false; + + let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); + + let err = resolve_identity( + &http::Method::GET, + "/test-bucket/key.txt", + "", + &headers, + &config, + None, ) .await .unwrap_err(); @@ -662,13 +787,185 @@ mod tests { }); } + // ── SigV4 spec compliance tests ────────────────────────────────── + + /// Validate our SigV4 implementation against the official AWS test suite. + /// Test vector: "get-vanilla" from + /// https://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html + #[test] + fn sigv4_test_vector_get_vanilla() { + let access_key_id = "AKIDEXAMPLE"; + let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + let date_stamp = "20150830"; + let amz_date = "20150830T123600Z"; + let region = "us-east-1"; + let service = "service"; + let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "example.amazonaws.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + + // Build the canonical request exactly as the spec defines: + // GET\n/\n\nhost:example.amazonaws.com\nx-amz-date:20150830T123600Z\n\nhost;x-amz-date\ne3b0c44... + let auth = SigV4Auth { + access_key_id: access_key_id.to_string(), + date_stamp: date_stamp.to_string(), + region: region.to_string(), + service: service.to_string(), + signed_headers: vec!["host".to_string(), "x-amz-date".to_string()], + signature: "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" + .to_string(), + }; + + let result = verify_sigv4_signature( + &http::Method::GET, + "/", + "", + &headers, + &auth, + secret, + payload_hash, + ) + .unwrap(); + + assert!(result, "AWS SigV4 test vector 'get-vanilla' must pass"); + } + + /// Test vector: "get-vanilla-query-order-key" — verifies query parameter sorting. + /// Parameters Param2 and Param1 must be sorted alphabetically. + #[test] + fn sigv4_test_vector_query_order() { + let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + let date_stamp = "20150830"; + let amz_date = "20150830T123600Z"; + let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "example.amazonaws.com".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + + let auth = SigV4Auth { + access_key_id: "AKIDEXAMPLE".to_string(), + date_stamp: date_stamp.to_string(), + region: "us-east-1".to_string(), + service: "service".to_string(), + signed_headers: vec!["host".to_string(), "x-amz-date".to_string()], + signature: "b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500" + .to_string(), + }; + + // Pass UNSORTED query — our canonicalization should sort to Param1=value1&Param2=value2 + let result = verify_sigv4_signature( + &http::Method::GET, + "/", + "Param2=value2&Param1=value1", + &headers, + &auth, + secret, + payload_hash, + ) + .unwrap(); + + assert!( + result, + "AWS SigV4 test vector 'get-vanilla-query-order-key' must pass" + ); + } + + /// Realistic S3 ListObjectsV2 request with host:port, security token, + /// and unsorted query parameters — mirrors what `aws s3 ls` sends. + #[test] + fn sigv4_list_objects_with_security_token_and_port() { + let secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; + let date_stamp = "20240101"; + let amz_date = "20240101T000000Z"; + let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + let mut headers = HeaderMap::new(); + headers.insert("host", "localhost:8787".parse().unwrap()); + headers.insert("x-amz-date", amz_date.parse().unwrap()); + headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + headers.insert("x-amz-security-token", session_token.parse().unwrap()); + + // Sign with sorted query (as AWS SDKs do) + let auth = sign_request( + &http::Method::GET, + "/private-uploads", + "list-type=2&prefix=&delimiter=%2F&encoding-type=url", + &headers, + "ASIATEMP1234EXAMPLE", + secret, + date_stamp, + amz_date, + "us-east-1", + &[ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-amz-security-token", + ], + payload_hash, + ); + headers.insert("authorization", auth.parse().unwrap()); + + // Verify with UNSORTED query (as it arrives from the raw URL) + let sig = parse_sigv4_auth( + headers + .get("authorization") + .unwrap() + .to_str() + .unwrap(), + ) + .unwrap(); + + let result = verify_sigv4_signature( + &http::Method::GET, + "/private-uploads", + "list-type=2&prefix=&delimiter=%2F&encoding-type=url", + &headers, + &sig, + secret, + payload_hash, + ) + .unwrap(); + + assert!(result, "S3 ListObjects with security token and host:port must verify"); + } + + // ── Sealed token tests ────────────────────────────────────────── + #[test] - fn temp_credential_wrong_signature_is_rejected() { + fn sealed_token_round_trip() { + use crate::sealed_token::TokenKey; + use crate::types::AccessScope; + run(async { - let real_secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; - let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; - let config = MockConfig::with_temp_credential(real_secret, session_token); + let key_bytes = [0x42u8; 32]; + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + key_bytes, + ); + let token_key = TokenKey::from_base64(&encoded).unwrap(); + + let secret = "TempSecretKey1234567890EXAMPLE000000000000"; + let creds = TemporaryCredentials { + access_key_id: "ASIATEMP1234EXAMPLE".into(), + secret_access_key: secret.into(), + session_token: String::new(), // will be replaced by seal + expiration: chrono::Utc::now() + chrono::Duration::hours(1), + allowed_scopes: vec![AccessScope { + bucket: "test-bucket".into(), + prefixes: vec![], + actions: vec![Action::GetObject], + }], + assumed_role_id: "role-1".into(), + source_identity: "test".into(), + }; + + let sealed = token_key.seal(&creds).unwrap(); + let config = MockConfig::empty(); let date_stamp = "20240101"; let amz_date = "20240101T000000Z"; @@ -678,16 +975,15 @@ mod tests { headers.insert("host", "s3.example.com".parse().unwrap()); headers.insert("x-amz-date", amz_date.parse().unwrap()); headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", session_token.parse().unwrap()); + headers.insert("x-amz-security-token", sealed.parse().unwrap()); - // Sign with wrong secret — session token is correct but sig won't match let auth = sign_request( &http::Method::GET, "/test-bucket/key.txt", "", &headers, "ASIATEMP1234EXAMPLE", - wrong_secret, + secret, date_stamp, amz_date, "us-east-1", @@ -701,44 +997,18 @@ mod tests { ); headers.insert("authorization", auth.parse().unwrap()); - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - ) - .await - .unwrap_err(); - - assert!( - matches!(err, ProxyError::SignatureDoesNotMatch), - "expected SignatureDoesNotMatch, got: {:?}", - err - ); - }); - } - - #[test] - fn disabled_credential_is_rejected_before_sig_check() { - run(async { - let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let mut config = MockConfig::with_credential(secret); - config.credentials[0].enabled = false; - - let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); - - let err = resolve_identity( + let identity = resolve_identity( &http::Method::GET, "/test-bucket/key.txt", "", &headers, &config, + Some(&token_key), ) .await - .unwrap_err(); + .unwrap(); - assert!(matches!(err, ProxyError::AccessDenied)); + assert!(matches!(identity, ResolvedIdentity::Temporary { .. })); }); } @@ -852,6 +1122,13 @@ pub fn authorize( if authorized { Ok(()) } else { + tracing::warn!( + action = ?action, + bucket = %bucket, + key = %key, + scopes = ?scopes, + "authorization denied — no scope grants access" + ); Err(ProxyError::AccessDenied) } } diff --git a/crates/libs/core/src/config/cached.rs b/crates/libs/core/src/config/cached.rs index 6e0e28e..dcd6d5b 100644 --- a/crates/libs/core/src/config/cached.rs +++ b/crates/libs/core/src/config/cached.rs @@ -22,7 +22,7 @@ use crate::config::ConfigProvider; use crate::error::ProxyError; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; @@ -43,8 +43,6 @@ impl CacheEntry { /// Wraps a [`ConfigProvider`] with in-memory TTL-based caching. /// /// Thread-safe via `RwLock`. Cache entries are evicted lazily on access. -/// Temporary credential storage is delegated directly to the underlying -/// provider (no caching for writes). #[derive(Clone)] pub struct CachedProvider

{ inner: P, @@ -201,22 +199,4 @@ impl ConfigProvider for CachedProvider

{ Ok(result) } - - /// Temporary credential writes bypass the cache and go directly to - /// the underlying provider. - async fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - self.inner.store_temporary_credential(cred).await - } - - /// Temporary credential reads also bypass the cache — they're already - /// short-lived and we don't want stale session tokens. - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - self.inner.get_temporary_credential(access_key_id).await - } } diff --git a/crates/libs/core/src/config/dynamodb.rs b/crates/libs/core/src/config/dynamodb.rs index 01171dc..d8fd454 100644 --- a/crates/libs/core/src/config/dynamodb.rs +++ b/crates/libs/core/src/config/dynamodb.rs @@ -12,7 +12,6 @@ //! | `BUCKET#{name}` | `CONFIG` | BucketConfig fields | //! | `ROLE#{role_id}` | `CONFIG` | RoleConfig fields | //! | `CRED#{access_key_id}` | `LONG_LIVED` | StoredCredential fields | -//! | `CRED#{access_key_id}` | `TEMPORARY` | TemporaryCredentials fields (with TTL) | //! //! # Example //! @@ -27,7 +26,7 @@ use crate::config::ConfigProvider; use crate::error::ProxyError; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use aws_sdk_dynamodb::types::AttributeValue; use aws_sdk_dynamodb::Client; use std::sync::Arc; @@ -168,64 +167,4 @@ impl ConfigProvider for DynamoDbProvider { } } - async fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - let json = serde_json::to_string(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; - - // TTL for DynamoDB auto-expiry - let ttl_epoch = cred.expiration.timestamp(); - - self.client() - .put_item() - .table_name(self.table()) - .item( - "PK", - AttributeValue::S(format!("CRED#{}", cred.access_key_id)), - ) - .item("SK", AttributeValue::S("TEMPORARY".into())) - .item("config_json", AttributeValue::S(json)) - .item("ttl", AttributeValue::N(ttl_epoch.to_string())) - .send() - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - Ok(()) - } - - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - let result = self - .client() - .get_item() - .table_name(self.table()) - .key("PK", AttributeValue::S(format!("CRED#{}", access_key_id))) - .key("SK", AttributeValue::S("TEMPORARY".into())) - .send() - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - match result.item() { - Some(item) => { - let json_val = item - .get("config_json") - .and_then(|v| v.as_s().ok()) - .ok_or_else(|| ProxyError::ConfigError("missing config_json".into()))?; - - let cred: TemporaryCredentials = serde_json::from_str(json_val) - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - // Check expiration - if cred.expiration <= chrono::Utc::now() { - return Ok(None); - } - - Ok(Some(cred)) - } - None => Ok(None), - } - } } diff --git a/crates/libs/core/src/config/http.rs b/crates/libs/core/src/config/http.rs index 2408217..ab2e5c9 100644 --- a/crates/libs/core/src/config/http.rs +++ b/crates/libs/core/src/config/http.rs @@ -10,8 +10,6 @@ //! - `GET /buckets/{name}` → `Option` //! - `GET /roles/{role_id}` → `Option` //! - `GET /credentials/{access_key_id}` → `Option` -//! - `POST /temporary-credentials` → stores a `TemporaryCredentials` -//! - `GET /temporary-credentials/{access_key_id}` → `Option` //! //! # Example //! @@ -26,7 +24,7 @@ use crate::config::ConfigProvider; use crate::error::ProxyError; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use std::sync::Arc; /// Validate that a value is safe to use as a single URL path segment. @@ -158,47 +156,6 @@ impl ConfigProvider for HttpProvider { .map_err(|e| ProxyError::ConfigError(e.to_string())) } - async fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - let mut req = self - .inner - .client - .post(format!("{}/temporary-credentials", self.inner.base_url)) - .json(cred); - - if let Some(ref auth) = self.inner.auth_header { - req = req.header("authorization", auth); - } - - req.send() - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - Ok(()) - } - - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - validate_path_segment(access_key_id, "access key ID")?; - let resp = self - .request(&format!("/temporary-credentials/{}", access_key_id)) - .send() - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - if resp.status().as_u16() == 404 { - return Ok(None); - } - - resp.json() - .await - .map(Some) - .map_err(|e| ProxyError::ConfigError(e.to_string())) - } } #[cfg(test)] diff --git a/crates/libs/core/src/config/mod.rs b/crates/libs/core/src/config/mod.rs index e300759..d766217 100644 --- a/crates/libs/core/src/config/mod.rs +++ b/crates/libs/core/src/config/mod.rs @@ -42,7 +42,7 @@ pub mod postgres; use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use std::future::Future; /// Trait for retrieving proxy configuration from a backend store. @@ -52,6 +52,9 @@ use std::future::Future; /// Methods use [`MaybeSend`] bounds — on native targets this resolves to `Send` /// (required by Tokio's task spawning), on WASM it's a no-op (allowing `!Send` /// JS interop types). +/// +/// Temporary credentials are not stored via this trait — they are encrypted +/// into self-contained session tokens using [`TokenKey`](crate::sealed_token::TokenKey). pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { fn list_buckets( &self, @@ -72,16 +75,4 @@ pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { &self, access_key_id: &str, ) -> impl Future, ProxyError>> + MaybeSend; - - /// Store a temporary credential (minted by the STS API). - fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> impl Future> + MaybeSend; - - /// Look up a temporary credential by its access key ID. - fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> impl Future, ProxyError>> + MaybeSend; } diff --git a/crates/libs/core/src/config/postgres.rs b/crates/libs/core/src/config/postgres.rs index 1a57153..6e90092 100644 --- a/crates/libs/core/src/config/postgres.rs +++ b/crates/libs/core/src/config/postgres.rs @@ -18,13 +18,8 @@ //! //! CREATE TABLE proxy_credentials ( //! access_key_id TEXT PRIMARY KEY, -//! credential_type TEXT NOT NULL, -- 'long_lived' or 'temporary' -//! config_json JSONB NOT NULL, -//! expires_at TIMESTAMPTZ +//! config_json JSONB NOT NULL //! ); -//! -//! CREATE INDEX idx_credentials_expires ON proxy_credentials(expires_at) -//! WHERE credential_type = 'temporary'; //! ``` //! //! # Example @@ -39,7 +34,7 @@ use crate::config::ConfigProvider; use crate::error::ProxyError; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use sqlx::PgPool; use std::sync::Arc; @@ -119,46 +114,4 @@ impl ConfigProvider for PostgresProvider { .transpose() } - async fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - let json = serde_json::to_value(cred).map_err(|e| ProxyError::Internal(e.to_string()))?; - - sqlx::query( - "INSERT INTO proxy_credentials (access_key_id, credential_type, config_json, expires_at) - VALUES ($1, 'temporary', $2, $3) - ON CONFLICT (access_key_id) DO UPDATE - SET config_json = EXCLUDED.config_json, expires_at = EXCLUDED.expires_at", - ) - .bind(&cred.access_key_id) - .bind(&json) - .bind(cred.expiration) - .execute(self.pool.as_ref()) - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - Ok(()) - } - - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - let row: Option<(serde_json::Value,)> = sqlx::query_as( - "SELECT config_json FROM proxy_credentials - WHERE access_key_id = $1 - AND credential_type = 'temporary' - AND expires_at > NOW()", - ) - .bind(access_key_id) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| ProxyError::ConfigError(e.to_string()))?; - - row.map(|(json,)| { - serde_json::from_value(json).map_err(|e| ProxyError::ConfigError(e.to_string())) - }) - .transpose() - } } diff --git a/crates/libs/core/src/config/static_file.rs b/crates/libs/core/src/config/static_file.rs index 63b9b57..0b99058 100644 --- a/crates/libs/core/src/config/static_file.rs +++ b/crates/libs/core/src/config/static_file.rs @@ -1,14 +1,13 @@ //! Static file-based configuration provider. //! -//! Loads configuration from a TOML or JSON file at startup. Stores temporary -//! credentials in memory. Suitable for simple deployments or development. +//! Loads configuration from a TOML or JSON file at startup. +//! Suitable for simple deployments or development. use crate::config::ConfigProvider; use crate::error::ProxyError; -use crate::types::{BucketConfig, RoleConfig, StoredCredential, TemporaryCredentials}; +use crate::types::{BucketConfig, RoleConfig, StoredCredential}; use serde::Deserialize; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; /// Full configuration file structure. #[derive(Debug, Clone, Deserialize)] @@ -48,8 +47,6 @@ pub struct StaticProvider { struct StaticProviderInner { config: StaticConfig, - /// In-memory store for temporary credentials. - temp_creds: RwLock>, } impl StaticProvider { @@ -80,10 +77,7 @@ impl StaticProvider { pub fn from_config(config: StaticConfig) -> Self { Self { - inner: Arc::new(StaticProviderInner { - config, - temp_creds: RwLock::new(HashMap::new()), - }), + inner: Arc::new(StaticProviderInner { config }), } } } @@ -126,43 +120,4 @@ impl ConfigProvider for StaticProvider { .cloned()) } - async fn store_temporary_credential( - &self, - cred: &TemporaryCredentials, - ) -> Result<(), ProxyError> { - let mut map = self - .inner - .temp_creds - .write() - .map_err(|e| ProxyError::Internal(e.to_string()))?; - - // Evict expired entries opportunistically - let now = chrono::Utc::now(); - map.retain(|_, v| v.expiration > now); - - map.insert(cred.access_key_id.clone(), cred.clone()); - Ok(()) - } - - async fn get_temporary_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - let map = self - .inner - .temp_creds - .read() - .map_err(|e| ProxyError::Internal(e.to_string()))?; - - let cred = map.get(access_key_id).cloned(); - - // Check expiration - if let Some(ref c) = cred { - if c.expiration <= chrono::Utc::now() { - return Ok(None); - } - } - - Ok(cred) - } } diff --git a/crates/libs/core/src/lib.rs b/crates/libs/core/src/lib.rs index 4fcfe2e..985ef42 100644 --- a/crates/libs/core/src/lib.rs +++ b/crates/libs/core/src/lib.rs @@ -27,4 +27,5 @@ pub mod proxy; pub mod resolver; pub mod response_body; pub mod s3; +pub mod sealed_token; pub mod types; diff --git a/crates/libs/core/src/resolver.rs b/crates/libs/core/src/resolver.rs index cbe1adc..ee3b977 100644 --- a/crates/libs/core/src/resolver.rs +++ b/crates/libs/core/src/resolver.rs @@ -11,6 +11,7 @@ use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; use crate::s3::request::{self, HostStyle}; use crate::s3::response::{BucketEntry, BucketList, BucketOwner, ListAllMyBucketsResult}; +use crate::sealed_token::TokenKey; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; @@ -65,13 +66,19 @@ pub struct ListRewrite { pub struct DefaultResolver

{ config: P, virtual_host_domain: Option, + token_key: Option, } impl

DefaultResolver

{ - pub fn new(config: P, virtual_host_domain: Option) -> Self { + pub fn new( + config: P, + virtual_host_domain: Option, + token_key: Option, + ) -> Self { Self { config, virtual_host_domain, + token_key, } } } @@ -138,9 +145,15 @@ impl RequestResolver for DefaultResolver

{ ); // Authenticate - let identity = - auth::resolve_identity(method, path, query.unwrap_or(""), headers, &self.config) - .await?; + let identity = auth::resolve_identity( + method, + path, + query.unwrap_or(""), + headers, + &self.config, + self.token_key.as_ref(), + ) + .await?; tracing::debug!(identity = ?identity, "resolved identity"); // Authorize diff --git a/crates/libs/core/src/sealed_token.rs b/crates/libs/core/src/sealed_token.rs new file mode 100644 index 0000000..6ecf509 --- /dev/null +++ b/crates/libs/core/src/sealed_token.rs @@ -0,0 +1,165 @@ +//! Self-contained encrypted session tokens using AES-256-GCM. +//! +//! When a `TokenKey` is configured, temporary credentials are encrypted into +//! the session token itself. The proxy decrypts the token on each request — +//! no server-side storage lookup is needed. This is critical for stateless +//! runtimes like Cloudflare Workers where in-memory state does not persist +//! across invocations. + +use crate::error::ProxyError; +use crate::types::TemporaryCredentials; +use aes_gcm::aead::{Aead, OsRng}; +use aes_gcm::{AeadCore, Aes256Gcm, KeyInit}; +use base64::Engine; +use std::sync::Arc; + +const NONCE_LEN: usize = 12; + +/// Wraps an AES-256-GCM cipher for sealing/unsealing session tokens. +#[derive(Clone)] +pub struct TokenKey(Arc); + +impl TokenKey { + /// Create a `TokenKey` from a base64-encoded 32-byte key. + pub fn from_base64(encoded: &str) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .map_err(|e| ProxyError::ConfigError(format!("invalid SESSION_TOKEN_KEY base64: {e}")))?; + if bytes.len() != 32 { + return Err(ProxyError::ConfigError(format!( + "SESSION_TOKEN_KEY must be 32 bytes, got {}", + bytes.len() + ))); + } + let cipher = Aes256Gcm::new_from_slice(&bytes) + .map_err(|e| ProxyError::ConfigError(format!("AES key error: {e}")))?; + Ok(Self(Arc::new(cipher))) + } + + /// Encrypt `TemporaryCredentials` into a base64url token. + /// + /// Format: `base64url(nonce[12] || ciphertext+tag)` + pub fn seal(&self, creds: &TemporaryCredentials) -> Result { + let plaintext = + serde_json::to_vec(creds).map_err(|e| ProxyError::Internal(format!("seal json: {e}")))?; + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ciphertext = self + .0 + .encrypt(&nonce, plaintext.as_slice()) + .map_err(|e| ProxyError::Internal(format!("seal encrypt: {e}")))?; + + let mut blob = Vec::with_capacity(NONCE_LEN + ciphertext.len()); + blob.extend_from_slice(&nonce); + blob.extend_from_slice(&ciphertext); + + Ok(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&blob)) + } + + /// Decrypt a session token back into `TemporaryCredentials`. + /// + /// Returns `Ok(None)` if the token doesn't look like a sealed token + /// (e.g. base64 decode fails or decryption fails — allows fallback to + /// config-based lookup). Returns `Err(ExpiredCredentials)` when the + /// token decrypts successfully but the credentials have expired. + pub fn unseal(&self, token: &str) -> Result, ProxyError> { + let blob = match base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(token) { + Ok(b) => b, + Err(_) => return Ok(None), + }; + + if blob.len() <= NONCE_LEN { + return Ok(None); + } + + let nonce = aes_gcm::Nonce::from_slice(&blob[..NONCE_LEN]); + let ciphertext = &blob[NONCE_LEN..]; + + let plaintext = match self.0.decrypt(nonce, ciphertext) { + Ok(p) => p, + Err(_) => return Ok(None), + }; + + let creds: TemporaryCredentials = serde_json::from_slice(&plaintext) + .map_err(|e| ProxyError::Internal(format!("unseal json: {e}")))?; + + if creds.expiration <= chrono::Utc::now() { + return Err(ProxyError::ExpiredCredentials); + } + + Ok(Some(creds)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::AccessScope; + + fn make_key() -> TokenKey { + let key_bytes = [0x42u8; 32]; + let encoded = base64::engine::general_purpose::STANDARD.encode(key_bytes); + TokenKey::from_base64(&encoded).unwrap() + } + + fn make_creds() -> TemporaryCredentials { + TemporaryCredentials { + access_key_id: "ASIATEMP".into(), + secret_access_key: "secret".into(), + session_token: "original-token".into(), + expiration: chrono::Utc::now() + chrono::Duration::hours(1), + allowed_scopes: vec![AccessScope { + bucket: "test-bucket".into(), + prefixes: vec![], + actions: vec![crate::types::Action::GetObject], + }], + assumed_role_id: "role-1".into(), + source_identity: "test".into(), + } + } + + #[test] + fn round_trip() { + let key = make_key(); + let creds = make_creds(); + let sealed = key.seal(&creds).unwrap(); + let unsealed = key.unseal(&sealed).unwrap().unwrap(); + assert_eq!(unsealed.access_key_id, creds.access_key_id); + assert_eq!(unsealed.secret_access_key, creds.secret_access_key); + assert_eq!(unsealed.assumed_role_id, creds.assumed_role_id); + } + + #[test] + fn wrong_key_returns_none() { + let key1 = make_key(); + let key2 = { + let key_bytes = [0x99u8; 32]; + let encoded = base64::engine::general_purpose::STANDARD.encode(key_bytes); + TokenKey::from_base64(&encoded).unwrap() + }; + let creds = make_creds(); + let sealed = key1.seal(&creds).unwrap(); + assert!(key2.unseal(&sealed).unwrap().is_none()); + } + + #[test] + fn non_sealed_token_returns_none() { + let key = make_key(); + assert!(key.unseal("FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng").unwrap().is_none()); + } + + #[test] + fn expired_token_returns_error() { + let key = make_key(); + let mut creds = make_creds(); + creds.expiration = chrono::Utc::now() - chrono::Duration::hours(1); + let sealed = key.seal(&creds).unwrap(); + let err = key.unseal(&sealed).unwrap_err(); + assert!(matches!(err, ProxyError::ExpiredCredentials)); + } + + #[test] + fn invalid_key_length_rejected() { + let short = base64::engine::general_purpose::STANDARD.encode([0u8; 16]); + assert!(TokenKey::from_base64(&short).is_err()); + } +} diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index fb2c032..1db7954 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -27,19 +27,33 @@ use request::StsRequest; pub use responses::{build_sts_error_response, build_sts_response}; use source_coop_core::config::ConfigProvider; use source_coop_core::error::ProxyError; +use source_coop_core::sealed_token::TokenKey; use source_coop_core::types::TemporaryCredentials; /// Try to handle an STS request. Returns `Some((status, xml))` if the query /// contained an STS action, or `None` if it wasn't an STS request. +/// +/// Requires a `TokenKey` — minted credentials are encrypted into the session +/// token itself, so no server-side storage is needed. If `token_key` is `None` +/// and an STS request arrives, an error response is returned. pub async fn try_handle_sts( query: Option<&str>, config: &C, jwks_cache: &JwksCache, + token_key: Option<&TokenKey>, ) -> Option<(u16, String)> { let sts_result = try_parse_sts_request(query)?; let (status, xml) = match sts_result { Ok(sts_request) => { - match assume_role_with_web_identity(config, &sts_request, "STSPRXY", jwks_cache).await { + let Some(key) = token_key else { + tracing::error!("STS request received but SESSION_TOKEN_KEY is not configured"); + return Some(build_sts_error_response(&ProxyError::ConfigError( + "STS requires SESSION_TOKEN_KEY to be configured".into(), + ))); + }; + match assume_role_with_web_identity(config, &sts_request, "STSPRXY", jwks_cache, key) + .await + { Ok(creds) => build_sts_response(&creds), Err(e) => { tracing::warn!(error = %e, "STS request failed"); @@ -76,11 +90,15 @@ fn jwt_decode_unverified( } /// Validate an OIDC token and mint temporary credentials. +/// +/// Credentials are encrypted into a self-contained session token via `token_key`. +/// No server-side credential storage is needed. pub async fn assume_role_with_web_identity( config: &C, sts_request: &StsRequest, key_prefix: &str, jwks_cache: &JwksCache, + token_key: &TokenKey, ) -> Result { // Look up the role let role = config @@ -146,10 +164,10 @@ pub async fn assume_role_with_web_identity( .unwrap_or(3600) .clamp(MIN_SESSION_DURATION_SECS, role.max_session_duration_secs); - let creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix, &claims); + let mut creds = sts::mint_temporary_credentials(&role, subject, duration, key_prefix, &claims); - // Store them - config.store_temporary_credential(&creds).await?; + // Encrypt the full credentials into the session token — stateless, no storage needed + creds.session_token = token_key.seal(&creds)?; Ok(creds) } diff --git a/crates/libs/sts/src/sts.rs b/crates/libs/sts/src/sts.rs index 5c65386..94fed6f 100644 --- a/crates/libs/sts/src/sts.rs +++ b/crates/libs/sts/src/sts.rs @@ -86,9 +86,9 @@ fn generate_random_id(len: usize) -> String { } fn generate_session_token() -> String { - // Real AWS session tokens are much longer; this is a simplified version - let id = Uuid::new_v4(); - format!("FwoGZXIvYXdzE{}", id.to_string().replace('-', "")) + use base64::Engine; + let bytes: Vec = (0..32).map(|_| rand_byte()).collect(); + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&bytes) } /// Simple random byte using UUID as entropy source (avoids extra deps). diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index f9690b1..27e85f0 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -36,6 +36,7 @@ use source_coop_core::proxy::{ ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, }; use source_coop_core::resolver::{DefaultResolver, RequestResolver}; +use source_coop_core::sealed_token::TokenKey; use source_coop_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; use axum::body::Body; @@ -67,6 +68,7 @@ async fn fetch( let reqwest_client = reqwest::Client::new(); let jwks_cache = JwksCache::new(reqwest_client.clone(), std::time::Duration::from_secs(900)); + let token_key = load_token_key(&env)?; let (parts, worker_body) = req.into_parts(); let body = Body::new(worker_body); @@ -80,7 +82,9 @@ async fn fetch( // STS uses STS_CONFIG (falling back to PROXY_CONFIG) for role definitions. if try_parse_sts_request(query.as_deref()).is_some() { let config = load_sts_config(&env)?; - if let Some((status, xml)) = try_handle_sts(query.as_deref(), &config, &jwks_cache).await { + if let Some((status, xml)) = + try_handle_sts(query.as_deref(), &config, &jwks_cache, token_key.as_ref()).await + { return Ok(Response::builder() .status(status) .header("content-type", "application/xml") @@ -132,7 +136,7 @@ async fn fetch( let config = load_static_config(&env)?; let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); - let resolver = DefaultResolver::new(config, virtual_host_domain); + let resolver = DefaultResolver::new(config, virtual_host_domain, token_key); let handler = ProxyHandler::new(WorkerBackend, resolver); Ok(handle_action( @@ -264,6 +268,18 @@ fn load_static_config(env: &Env) -> Result { load_config_from_env(env, "PROXY_CONFIG") } +/// Load the optional session token encryption key from the `SESSION_TOKEN_KEY` secret. +fn load_token_key(env: &Env) -> Result> { + match env.secret("SESSION_TOKEN_KEY") { + Ok(val) => { + let key = TokenKey::from_base64(&val.to_string()) + .map_err(|e| worker::Error::RustError(e.to_string()))?; + Ok(Some(key)) + } + Err(_) => Ok(None), + } +} + /// Load STS config: tries STS_CONFIG first, falls back to PROXY_CONFIG. fn load_sts_config(env: &Env) -> Result { load_config_from_env(env, "STS_CONFIG").or_else(|_| load_config_from_env(env, "PROXY_CONFIG")) diff --git a/crates/runtimes/cf-workers/wrangler.toml b/crates/runtimes/cf-workers/wrangler.toml index 03e59d7..3081e56 100644 --- a/crates/runtimes/cf-workers/wrangler.toml +++ b/crates/runtimes/cf-workers/wrangler.toml @@ -6,7 +6,7 @@ name = "source-coop-proxy" command = "cargo install worker-build && worker-build --release" [vars] -SOURCE_API_URL = "https://staging.source.coop" +# SOURCE_API_URL = "https://staging.source.coop" VIRTUAL_HOST_DOMAIN = "s3.local" # For production, consider storing this in Workers KV or a Secrets binding. @@ -104,4 +104,9 @@ trusted_oidc_issuers = [ [[vars.STS_CONFIG.roles.allowed_scopes]] actions = ["get_object", "head_object", "put_object", "list_bucket"] bucket = "{sub}" # this user +prefixes = [] + +[[vars.STS_CONFIG.roles.allowed_scopes]] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +bucket = "private-uploads" prefixes = [] diff --git a/crates/runtimes/server/src/bin/source-coop-proxy.rs b/crates/runtimes/server/src/bin/source-coop-proxy.rs index cf12946..4421c73 100644 --- a/crates/runtimes/server/src/bin/source-coop-proxy.rs +++ b/crates/runtimes/server/src/bin/source-coop-proxy.rs @@ -5,6 +5,7 @@ use source_coop_core::config::cached::CachedProvider; use source_coop_core::config::static_file::StaticProvider; +use source_coop_core::sealed_token::TokenKey; use source_coop_server::server::{run, ServerConfig}; use std::net::SocketAddr; use std::time::Duration; @@ -60,9 +61,15 @@ async fn main() -> Result<(), Box> { let config = CachedProvider::new(base_config, Duration::from_secs(60)); let sts_config = CachedProvider::new(sts_base, Duration::from_secs(60)); + let token_key = std::env::var("SESSION_TOKEN_KEY") + .ok() + .map(|v| TokenKey::from_base64(&v)) + .transpose()?; + let server_config = ServerConfig { listen_addr, virtual_host_domain: domain, + token_key, }; run(config, sts_config, server_config).await diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index f6a834b..743920b 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -14,6 +14,7 @@ use source_coop_core::proxy::{ ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, }; use source_coop_core::resolver::DefaultResolver; +use source_coop_core::sealed_token::TokenKey; use source_coop_sts::{try_handle_sts, JwksCache}; use std::net::SocketAddr; use std::sync::Arc; @@ -26,6 +27,8 @@ pub struct ServerConfig { /// The base domain for virtual-hosted-style requests (e.g., "s3.example.com"). /// If set, requests to `{bucket}.s3.example.com` use virtual-hosted style. pub virtual_host_domain: Option, + /// Optional AES-256-GCM key for self-contained encrypted session tokens. + pub token_key: Option, } impl Default for ServerConfig { @@ -33,6 +36,7 @@ impl Default for ServerConfig { Self { listen_addr: ([0, 0, 0, 0], 8080).into(), virtual_host_domain: None, + token_key: None, } } } @@ -42,6 +46,7 @@ struct AppState { reqwest_client: reqwest::Client, sts_config: P, jwks_cache: JwksCache, + token_key: Option, } /// Run the S3 proxy server. @@ -59,6 +64,7 @@ struct AppState { /// let server_config = ServerConfig { /// listen_addr: ([0, 0, 0, 0], 8080).into(), /// virtual_host_domain: Some("s3.local".to_string()), +/// ..Default::default() /// }; /// run(config, sts_config, server_config).await.unwrap(); /// } @@ -74,7 +80,8 @@ where let backend = ServerBackend::new(); let reqwest_client = backend.client().clone(); let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); - let resolver = DefaultResolver::new(config, server_config.virtual_host_domain); + let token_key = server_config.token_key; + let resolver = DefaultResolver::new(config, server_config.virtual_host_domain, token_key.clone()); let handler = ProxyHandler::new(backend, resolver); let state = Arc::new(AppState { @@ -82,6 +89,7 @@ where reqwest_client, sts_config, jwks_cache, + token_key, }); let app = Router::new() @@ -114,7 +122,7 @@ async fn request_handler( // Intercept STS AssumeRoleWithWebIdentity requests if let Some((status, xml)) = - try_handle_sts(query.as_deref(), &state.sts_config, &state.jwks_cache).await + try_handle_sts(query.as_deref(), &state.sts_config, &state.jwks_cache, state.token_key.as_ref()).await { return Response::builder() .status(status) From c941f01561defd11871cbdb9f4a0508a26b91304 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 27 Feb 2026 15:36:13 -0500 Subject: [PATCH 53/82] feat: wire up oidc-provider tooling --- CLAUDE.md | 8 +- Cargo.lock | 2 + README.md | 2 + crates/libs/core/README.md | 7 +- crates/libs/core/src/backend.rs | 13 +- crates/libs/core/src/lib.rs | 1 + crates/libs/core/src/oidc_backend.rs | 47 ++++ crates/libs/core/src/proxy.rs | 35 ++- crates/libs/oidc-provider/src/backend_auth.rs | 253 ++++++++++++++++++ crates/libs/oidc-provider/src/lib.rs | 1 + crates/runtimes/cf-workers/Cargo.toml | 1 + crates/runtimes/cf-workers/README.md | 8 +- crates/runtimes/cf-workers/src/client.rs | 25 ++ crates/runtimes/cf-workers/src/lib.rs | 83 +++++- crates/runtimes/server/Cargo.toml | 1 + crates/runtimes/server/README.md | 7 +- .../server/src/bin/source-coop-proxy.rs | 5 + crates/runtimes/server/src/client.rs | 33 +++ crates/runtimes/server/src/server.rs | 74 ++++- 19 files changed, 592 insertions(+), 14 deletions(-) create mode 100644 crates/libs/core/src/oidc_backend.rs create mode 100644 crates/libs/oidc-provider/src/backend_auth.rs diff --git a/CLAUDE.md b/CLAUDE.md index f87b332..0567107 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ cargo test - `Forward(ForwardRequest)` — presigned URL + headers for GET/HEAD/PUT/DELETE. The runtime executes the request with its native HTTP client, enabling zero-copy streaming. - `Response(ProxyResult)` — complete response for LIST, errors, synthetic responses. - `NeedsBody(PendingRequest)` — multipart operations that need the request body. The runtime materializes the body and calls `handle_with_body()`. -- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend` and `R: RequestResolver`. The backend trait has three methods: `create_store()` returns an `Arc` for LIST, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: +- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend`, `R: RequestResolver`, and `O: OidcBackendAuth` (defaults to `NoOidcAuth`). The backend trait has three methods: `create_store()` returns an `Arc` for LIST, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: - **Server**: `ServerBackend` delegates to `build_object_store()` (with default connector) and `build_signer()`, and uses reqwest for raw HTTP + Forward execution. - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` (injecting `FetchConnector`) and `build_signer()`, and uses `web_sys::fetch` for raw HTTP + Forward execution. - **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_object_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_object_store()` only — `build_signer()` needs no connector since signing is pure computation. @@ -46,10 +46,16 @@ cargo test - **Streaming via Forward pattern**: For GET, the runtime sends a presigned URL request and streams the response body directly to the client. For PUT, the runtime streams the client's request body directly to the presigned URL. On CF Workers, JS `ReadableStream` objects pass through without touching Rust. On the server, reqwest streams hyper `Incoming` bodies and `bytes_stream()` responses. - **cf-workers is excluded from `default-members`** in the root `Cargo.toml` because WASM types are `!Send` and will fail to compile on native targets. Always use `--target wasm32-unknown-unknown` when working with this crate. - **Config loading** (CF Workers): `PROXY_CONFIG` can be either a JSON string (via `wrangler secret`) or a JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`). Both formats are handled. +- **Sealed session tokens**: When `SESSION_TOKEN_KEY` is configured, temporary credentials minted by STS are AES-256-GCM encrypted into the session token itself (`sealed_token.rs`). On subsequent requests, `resolve_identity()` decrypts the token to recover credentials — no server-side storage or config lookup needed. This is required for stateless runtimes (CF Workers). `TokenKey` wraps `Arc` (Clone + Send + Sync). Token format: `base64url(nonce[12] || ciphertext + tag)`. Scopes are sealed at mint time, so config changes to `allowed_scopes` only affect newly minted credentials. The `DefaultResolver` accepts an optional `TokenKey` as its third constructor argument; the STS handler requires it when processing STS requests. - **List response construction**: LIST responses are built directly from `object_store::ListResult` as S3 XML. When a resolver returns a `ListRewrite`, prefix stripping/adding is applied to `ObjectMeta.location` and `common_prefixes` paths before XML generation. The `list_rewrite` module in `crates/libs/core/src/s3/list_rewrite.rs` is retained for backward compatibility. +- **OIDC backend auth**: The `OidcBackendAuth` trait (`crates/libs/core/src/oidc_backend.rs`) resolves backend credentials via OIDC token exchange. When a bucket's `backend_options` contains `auth_type=oidc`, the proxy mints a self-signed JWT and exchanges it for temporary cloud credentials before the request reaches `create_store()`/`create_signer()`. The resolved credentials are injected into a cloned `BucketConfig.backend_options` so the existing builder pipeline works unmodified. `AwsOidcBackendAuth` (in `crates/libs/oidc-provider/src/backend_auth.rs`) implements this for AWS via `AssumeRoleWithWebIdentity`. `MaybeOidcAuth` is an enum (`Enabled`/`Disabled`) used as the concrete `O` type by both runtimes. OIDC is configured via `OIDC_PROVIDER_KEY` (PEM secret) and `OIDC_PROVIDER_ISSUER` (URL). When configured, `/.well-known/openid-configuration` and `/.well-known/jwks.json` are served for cloud provider JWKS discovery. The `S3RequestSigner` includes `x-amz-security-token` for STS temporary credentials. Currently AWS/S3 only; Azure and GCP exchange flows are TODO. ## Known Limitations 1. **Multipart uses raw HTTP (S3 only)**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP. They are gated to `backend_type == "s3"` — non-S3 backends return an error for multipart requests and should use `PUT` (object_store handles chunking internally). 2. **LIST returns all results**: `object_store::list_with_delimiter()` fetches all pages internally. No S3-style pagination (continuation tokens, max-keys truncation). `IsTruncated` is always `false`. 3. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. + +## Style + +Don't support anything legacy. \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index c7433cf..045c901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2667,6 +2667,7 @@ dependencies = [ "serde_json", "source-coop-api", "source-coop-core", + "source-coop-oidc-provider", "source-coop-sts", "thiserror", "tracing", @@ -2759,6 +2760,7 @@ dependencies = [ "serde", "source-coop-api", "source-coop-core", + "source-coop-oidc-provider", "source-coop-sts", "thiserror", "tokio", diff --git a/README.md b/README.md index 174fe2d..40c5bdd 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,8 @@ The proxy validates the JWT against the OIDC provider's JWKS, checks the trust p openssl rand -base64 32 ``` +**OIDC backend auth:** When `OIDC_PROVIDER_KEY` (PEM-encoded RSA private key) and `OIDC_PROVIDER_ISSUER` (publicly reachable URL) are configured, the proxy acts as its own OIDC identity provider for backend authentication. Buckets configured with `auth_type=oidc` and `oidc_role_arn` in their `backend_options` will have credentials resolved automatically — the proxy mints a self-signed JWT, exchanges it with the cloud provider's STS for temporary credentials, and caches them. This eliminates the need to store long-lived backend credentials. The proxy serves `/.well-known/openid-configuration` and `/.well-known/jwks.json` for cloud provider JWKS discovery. Currently supports AWS S3 only. + ## Multi-Runtime Design The crate workspace separates concerns so the core logic compiles to both native and WASM targets: diff --git a/crates/libs/core/README.md b/crates/libs/core/README.md index ebe0ac4..c9c72b9 100644 --- a/crates/libs/core/README.md +++ b/crates/libs/core/README.md @@ -8,7 +8,7 @@ The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in conta ## Key Abstractions -The core defines three trait boundaries that runtime crates implement: +The core defines four trait boundaries that runtime crates implement: **`ProxyBackend`** — Provides three capabilities: `create_store()` returns an `ObjectStore` for LIST, `create_signer()` returns a `Signer` for presigned URL generation (GET/HEAD/PUT/DELETE), and `send_raw()` sends signed HTTP requests for multipart operations. Both runtimes delegate to `build_signer()` which uses `object_store`'s built-in signer for authenticated backends and `UnsignedUrlSigner` for anonymous backends (avoiding `Instant::now()` which panics on WASM). For `create_store()`, the server runtime uses default connectors + reqwest; the worker runtime uses a custom `FetchConnector`. @@ -27,6 +27,8 @@ Any provider can be wrapped with `CachedProvider` for in-memory TTL caching. `DefaultResolver` implements the standard S3 proxy flow: parse the S3 operation, look up the bucket in config, authenticate via SigV4, and authorize. Custom resolvers (like the Source Cooperative resolver in `cf-workers`) can implement entirely different routing and auth schemes. +**`OidcBackendAuth`** — Resolves backend credentials via OIDC token exchange. Called at the top of `dispatch_operation()` before the config reaches `create_store()`/`create_signer()`. When a bucket's `backend_options` contains `auth_type=oidc`, the implementation mints a self-signed JWT and exchanges it for temporary cloud credentials, injecting them into the config. The default `NoOidcAuth` passes configs through unchanged (and errors if `auth_type=oidc` is set without a provider). The `oidc-provider` crate provides `AwsOidcBackendAuth` and `MaybeOidcAuth` as concrete implementations. + ## Module Overview ``` @@ -41,6 +43,7 @@ src/ │ ├── dynamodb.rs DynamoDB provider (feature: config-dynamodb) │ └── postgres.rs PostgreSQL provider (feature: config-postgres) ├── error.rs ProxyError with S3-compatible error codes +├── oidc_backend.rs OidcBackendAuth trait, NoOidcAuth default impl ├── proxy.rs ProxyHandler — the main request handler ├── resolver.rs RequestResolver trait, ResolvedAction, DefaultResolver ├── sealed_token.rs AES-256-GCM encrypted session tokens (TokenKey) @@ -74,6 +77,8 @@ let token_key = None; // or Some(TokenKey::from_base64(&key_b64)?) let resolver = DefaultResolver::new(config, Some("s3.example.com".into()), token_key); let handler = ProxyHandler::new(backend, resolver); +// Optional: enable OIDC-based backend credential resolution. +// let handler = handler.with_oidc_auth(oidc_auth); // In your HTTP handler: let action = handler.resolve_request(method, path, query, &headers).await; diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index 4dfe211..b7e46d1 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -233,15 +233,22 @@ pub struct S3RequestSigner { pub secret_access_key: String, pub region: String, pub service: String, + pub session_token: Option, } impl S3RequestSigner { - pub fn new(access_key_id: String, secret_access_key: String, region: String) -> Self { + pub fn new( + access_key_id: String, + secret_access_key: String, + region: String, + session_token: Option, + ) -> Self { Self { access_key_id, secret_access_key, region, service: "s3".to_string(), + session_token, } } @@ -268,6 +275,10 @@ impl S3RequestSigner { headers.insert("x-amz-date", amz_date.parse().unwrap()); headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + if let Some(token) = &self.session_token { + headers.insert("x-amz-security-token", token.parse().unwrap()); + } + let host = url .host_str() .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; diff --git a/crates/libs/core/src/lib.rs b/crates/libs/core/src/lib.rs index 985ef42..0daf967 100644 --- a/crates/libs/core/src/lib.rs +++ b/crates/libs/core/src/lib.rs @@ -23,6 +23,7 @@ pub mod backend; pub mod config; pub mod error; pub mod maybe_send; +pub mod oidc_backend; pub mod proxy; pub mod resolver; pub mod response_body; diff --git a/crates/libs/core/src/oidc_backend.rs b/crates/libs/core/src/oidc_backend.rs new file mode 100644 index 0000000..d9d0f76 --- /dev/null +++ b/crates/libs/core/src/oidc_backend.rs @@ -0,0 +1,47 @@ +//! Trait for OIDC-based backend credential resolution. +//! +//! When a bucket is configured with `auth_type=oidc`, the proxy mints a +//! self-signed JWT and exchanges it with the cloud provider's STS for +//! temporary credentials. The resolved credentials are injected into the +//! `BucketConfig.backend_options` so the existing builder pipeline works +//! unmodified. +//! +//! [`NoOidcAuth`] is the default no-op implementation used when no OIDC +//! provider is configured. + +use crate::error::ProxyError; +use crate::maybe_send::MaybeSend; +use crate::types::BucketConfig; +use std::future::Future; + +/// Resolves backend credentials via OIDC token exchange. +/// +/// Called at the top of `dispatch_operation()` before the config reaches +/// `create_store()` / `create_signer()`. Implementations may return the +/// config unchanged (no `auth_type=oidc`) or inject temporary credentials. +pub trait OidcBackendAuth: MaybeSend + 'static { + fn resolve_credentials( + &self, + config: &BucketConfig, + ) -> impl Future> + MaybeSend; +} + +/// No-op implementation — returns config unchanged. +/// +/// If a bucket specifies `auth_type=oidc` but no OIDC provider is +/// configured, this returns a `ConfigError`. +pub struct NoOidcAuth; + +impl OidcBackendAuth for NoOidcAuth { + async fn resolve_credentials( + &self, + config: &BucketConfig, + ) -> Result { + if config.option("auth_type") == Some("oidc") { + return Err(ProxyError::ConfigError( + "bucket requires auth_type=oidc but no OIDC provider is configured".into(), + )); + } + Ok(config.clone()) + } +} diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 3d6f765..b46d68a 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -16,6 +16,7 @@ use crate::backend::{hash_payload, ProxyBackend, S3RequestSigner, UNSIGNED_PAYLOAD}; use crate::error::ProxyError; +use crate::oidc_backend::{NoOidcAuth, OidcBackendAuth}; use crate::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use crate::response_body::ProxyResponseBody; use crate::s3::pagination::{self, paginate}; @@ -68,9 +69,11 @@ pub struct PendingRequest { /// /// - `B`: The runtime's backend for object store creation, signing, and raw HTTP /// - `R`: The request resolver that decides what action to take for each request -pub struct ProxyHandler { +/// - `O`: OIDC backend auth for resolving credentials via token exchange +pub struct ProxyHandler { backend: B, resolver: R, + oidc_auth: O, /// When true, error responses include full internal details (for development). /// When false, server-side errors use generic messages. debug_errors: bool, @@ -85,9 +88,31 @@ where Self { backend, resolver, + oidc_auth: NoOidcAuth, debug_errors: false, } } +} + +impl ProxyHandler +where + B: ProxyBackend, + R: RequestResolver, + O: OidcBackendAuth, +{ + /// Set the OIDC backend auth implementation. + /// + /// When configured, `dispatch_operation` calls `resolve_credentials` + /// before accessing the backend — enabling OIDC-based credential + /// resolution for buckets with `auth_type=oidc`. + pub fn with_oidc_auth(self, oidc_auth: O2) -> ProxyHandler { + ProxyHandler { + backend: self.backend, + resolver: self.resolver, + oidc_auth, + debug_errors: self.debug_errors, + } + } /// Enable verbose error messages in S3 error responses. /// @@ -244,6 +269,12 @@ where list_rewrite: Option<&ListRewrite>, request_id: &str, ) -> Result { + // Resolve OIDC credentials if auth_type=oidc is configured. + // This injects temporary credentials into a cloned config so the + // existing builder pipeline works unmodified. + let bucket_config = self.oidc_auth.resolve_credentials(bucket_config).await?; + let bucket_config = &bucket_config; + match operation { S3Operation::GetObject { key, .. } => { let fwd = self @@ -534,10 +565,12 @@ fn sign_s3_request( Url::parse(url).map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; if has_credentials { + let session_token = config.option("token").map(|s| s.to_string()); let signer = S3RequestSigner::new( access_key.to_string(), secret_key.to_string(), region.to_string(), + session_token, ); signer.sign_request(method, &parsed_url, headers, payload_hash)?; } else { diff --git a/crates/libs/oidc-provider/src/backend_auth.rs b/crates/libs/oidc-provider/src/backend_auth.rs new file mode 100644 index 0000000..19f944c --- /dev/null +++ b/crates/libs/oidc-provider/src/backend_auth.rs @@ -0,0 +1,253 @@ +//! OIDC-based backend credential resolution. +//! +//! When a bucket's `backend_options` contains `auth_type=oidc`, the proxy +//! mints a self-signed JWT and exchanges it for temporary cloud credentials +//! via the cloud provider's STS. The resolved credentials are injected back +//! into the config so the existing builder pipeline works unmodified. + +use source_coop_core::error::ProxyError; +use source_coop_core::oidc_backend::OidcBackendAuth; +use source_coop_core::types::BucketConfig; + +use crate::exchange::aws::AwsExchange; +use crate::{HttpExchange, OidcCredentialProvider}; + +/// AWS OIDC backend auth — exchanges a self-signed JWT for temporary +/// AWS credentials via `AssumeRoleWithWebIdentity`. +pub struct AwsOidcBackendAuth { + provider: OidcCredentialProvider, +} + +impl AwsOidcBackendAuth { + pub fn new(provider: OidcCredentialProvider) -> Self { + Self { provider } + } + + async fn resolve_aws( + &self, + config: &BucketConfig, + ) -> Result { + let role_arn = config.option("oidc_role_arn").ok_or_else(|| { + ProxyError::ConfigError("auth_type=oidc requires 'oidc_role_arn' in backend_options".into()) + })?; + let subject = config + .option("oidc_subject") + .unwrap_or("s3-proxy"); + + let exchange = AwsExchange::new(role_arn.to_string()); + let creds = self + .provider + .get_credentials(role_arn, &exchange, subject, &[]) + .await?; + + let mut resolved = config.clone(); + resolved + .backend_options + .insert("access_key_id".into(), creds.access_key_id.clone()); + resolved + .backend_options + .insert("secret_access_key".into(), creds.secret_access_key.clone()); + resolved + .backend_options + .insert("token".into(), creds.session_token.clone()); + + // Remove OIDC-specific keys so they don't confuse the builder. + resolved.backend_options.remove("auth_type"); + resolved.backend_options.remove("oidc_role_arn"); + resolved.backend_options.remove("oidc_subject"); + + Ok(resolved) + } +} + +impl OidcBackendAuth for AwsOidcBackendAuth { + async fn resolve_credentials( + &self, + config: &BucketConfig, + ) -> Result { + if config.option("auth_type") != Some("oidc") { + return Ok(config.clone()); + } + + // TODO: dispatch on backend_type for Azure/GCP when those exchanges are wired up. + match config.backend_type.as_str() { + "s3" => self.resolve_aws(config).await, + other => Err(ProxyError::ConfigError(format!( + "OIDC backend auth not yet supported for backend_type '{other}'" + ))), + } + } +} + +/// Wrapper enum that runtimes use as a single concrete `O` type. +/// +/// `Enabled` holds the live OIDC provider; `Disabled` is the no-op fallback. +/// When disabled and a bucket specifies `auth_type=oidc`, a `ConfigError` +/// is returned (same as `NoOidcAuth`). +pub enum MaybeOidcAuth { + Enabled(AwsOidcBackendAuth), + Disabled, +} + +impl OidcBackendAuth for MaybeOidcAuth { + async fn resolve_credentials( + &self, + config: &BucketConfig, + ) -> Result { + match self { + MaybeOidcAuth::Enabled(auth) => auth.resolve_credentials(config).await, + MaybeOidcAuth::Disabled => { + if config.option("auth_type") == Some("oidc") { + Err(ProxyError::ConfigError( + "bucket requires auth_type=oidc but no OIDC provider is configured".into(), + )) + } else { + Ok(config.clone()) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::jwt::JwtSigner; + use crate::OidcProviderError; + use chrono::{Duration, Utc}; + use std::collections::HashMap; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[derive(Clone)] + struct MockHttp { + call_count: Arc, + } + + impl MockHttp { + fn new() -> Self { + Self { + call_count: Arc::new(AtomicUsize::new(0)), + } + } + } + + impl HttpExchange for MockHttp { + async fn post_form( + &self, + _url: &str, + _form: &[(&str, &str)], + ) -> Result { + self.call_count.fetch_add(1, Ordering::SeqCst); + let exp = (Utc::now() + Duration::hours(1)).to_rfc3339(); + Ok(format!( + r#" + + + AKID_OIDC + secret_oidc + token_oidc + {exp} + + + "# + )) + } + } + + fn test_signer() -> JwtSigner { + use rsa::pkcs8::EncodePrivateKey; + let mut rng = rand::rngs::OsRng; + let key = rsa::RsaPrivateKey::new(&mut rng, 2048).unwrap(); + let pem = key.to_pkcs8_pem(rsa::pkcs8::LineEnding::LF).unwrap(); + JwtSigner::from_pem(&pem, "test-kid".into(), 300).unwrap() + } + + fn oidc_bucket_config() -> BucketConfig { + let mut opts = HashMap::new(); + opts.insert("auth_type".into(), "oidc".into()); + opts.insert("oidc_role_arn".into(), "arn:aws:iam::123:role/Test".into()); + opts.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + opts.insert("bucket_name".into(), "my-bucket".into()); + opts.insert("region".into(), "us-east-1".into()); + BucketConfig { + name: "test".into(), + backend_type: "s3".into(), + backend_prefix: None, + anonymous_access: false, + allowed_roles: vec![], + backend_options: opts, + } + } + + fn static_bucket_config() -> BucketConfig { + let mut opts = HashMap::new(); + opts.insert("access_key_id".into(), "AKID_STATIC".into()); + opts.insert("secret_access_key".into(), "secret_static".into()); + opts.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + opts.insert("bucket_name".into(), "my-bucket".into()); + BucketConfig { + name: "test".into(), + backend_type: "s3".into(), + backend_prefix: None, + anonymous_access: false, + allowed_roles: vec![], + backend_options: opts, + } + } + + #[tokio::test] + async fn resolve_injects_creds_for_oidc_bucket() { + let http = MockHttp::new(); + let provider = OidcCredentialProvider::new( + test_signer(), + http, + "https://issuer.example.com".into(), + "sts.amazonaws.com".into(), + ); + let auth = AwsOidcBackendAuth::new(provider); + + let config = oidc_bucket_config(); + let resolved = auth.resolve_credentials(&config).await.unwrap(); + + assert_eq!(resolved.option("access_key_id"), Some("AKID_OIDC")); + assert_eq!(resolved.option("secret_access_key"), Some("secret_oidc")); + assert_eq!(resolved.option("token"), Some("token_oidc")); + assert!(resolved.option("auth_type").is_none()); + assert!(resolved.option("oidc_role_arn").is_none()); + } + + #[tokio::test] + async fn resolve_passes_through_static_bucket() { + let http = MockHttp::new(); + let provider = OidcCredentialProvider::new( + test_signer(), + http.clone(), + "https://issuer.example.com".into(), + "sts.amazonaws.com".into(), + ); + let auth = AwsOidcBackendAuth::new(provider); + + let config = static_bucket_config(); + let resolved = auth.resolve_credentials(&config).await.unwrap(); + + assert_eq!(resolved.option("access_key_id"), Some("AKID_STATIC")); + assert_eq!(http.call_count.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn maybe_disabled_errors_on_oidc_bucket() { + let auth: MaybeOidcAuth = MaybeOidcAuth::Disabled; + let config = oidc_bucket_config(); + let err = auth.resolve_credentials(&config).await.unwrap_err(); + assert!(err.to_string().contains("no OIDC provider is configured")); + } + + #[tokio::test] + async fn maybe_disabled_passes_through_static_bucket() { + let auth: MaybeOidcAuth = MaybeOidcAuth::Disabled; + let config = static_bucket_config(); + let resolved = auth.resolve_credentials(&config).await.unwrap(); + assert_eq!(resolved.option("access_key_id"), Some("AKID_STATIC")); + } +} diff --git a/crates/libs/oidc-provider/src/lib.rs b/crates/libs/oidc-provider/src/lib.rs index 67b6816..865d3b1 100644 --- a/crates/libs/oidc-provider/src/lib.rs +++ b/crates/libs/oidc-provider/src/lib.rs @@ -12,6 +12,7 @@ //! [`HttpExchange`] trait so that each runtime (reqwest, Fetch API, etc.) //! can provide its own implementation. +pub mod backend_auth; pub mod cache; pub mod discovery; pub mod exchange; diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index fffdb3e..a8f9f2f 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] [dependencies] source-coop-core = { workspace = true, features = ["axum"] } source-coop-sts.workspace = true +source-coop-oidc-provider.workspace = true source-coop-api.workspace = true axum = { workspace = true, features = ["json"] } bytes.workspace = true diff --git a/crates/runtimes/cf-workers/README.md b/crates/runtimes/cf-workers/README.md index 749f13e..cde3653 100644 --- a/crates/runtimes/cf-workers/README.md +++ b/crates/runtimes/cf-workers/README.md @@ -25,7 +25,7 @@ Client request src/ ├── lib.rs Worker entry point, two-phase request handling, Forward execution ├── body.rs ProxyResult → worker::Response conversion (Bytes/Empty only) -├── client.rs WorkerBackend implementing ProxyBackend, WorkerHttpClient +├── client.rs WorkerBackend implementing ProxyBackend, WorkerHttpClient, FetchHttpExchange ├── fetch_connector.rs FetchConnector/FetchService bridging object_store to Fetch API (LIST only) └── tracing_layer.rs Minimal tracing subscriber for Workers console_log ``` @@ -41,6 +41,10 @@ Reads bucket configuration from the `PROXY_CONFIG` environment variable. Uses `D [vars] PROXY_CONFIG = '{"buckets":[...],"roles":[...],"credentials":[...]}' VIRTUAL_HOST_DOMAIN = "s3.example.com" # optional, for virtual-hosted style +OIDC_PROVIDER_ISSUER = "https://data.example.com" # optional, for OIDC backend auth + +# Set via wrangler secret (PEM-encoded RSA private key): +# wrangler secret put OIDC_PROVIDER_KEY ``` ### Source Cooperative Mode @@ -58,9 +62,11 @@ Authentication is handled by the Source API permissions endpoint rather than the # wrangler.toml [vars] SOURCE_API_URL = "https://api.source.coop" +OIDC_PROVIDER_ISSUER = "https://data.source.coop" # optional, for OIDC backend auth # Set via wrangler secret: # wrangler secret put SOURCE_API_KEY +# wrangler secret put OIDC_PROVIDER_KEY # optional, PEM-encoded RSA private key ``` ### Implementing a Custom Resolver diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 215d2cc..383b66e 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -16,6 +16,7 @@ use source_coop_core::backend::{ }; use source_coop_core::error::ProxyError; use source_coop_core::types::BucketConfig; +use source_coop_oidc_provider::{HttpExchange, OidcProviderError}; use std::sync::Arc; use worker::{Cache, Fetch}; @@ -227,3 +228,27 @@ pub fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { } resp_headers } + +/// [`HttpExchange`] implementation using reqwest on WASM (wraps `web_sys::fetch`). +#[derive(Clone)] +pub struct FetchHttpExchange; + +impl HttpExchange for FetchHttpExchange { + async fn post_form( + &self, + url: &str, + form: &[(&str, &str)], + ) -> Result { + let client = reqwest::Client::new(); + let resp = client + .post(url) + .form(form) + .send() + .await + .map_err(|e| OidcProviderError::HttpError(e.to_string()))?; + + resp.text() + .await + .map_err(|e| OidcProviderError::HttpError(e.to_string())) + } +} diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 27e85f0..828b49d 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -27,16 +27,20 @@ mod client; mod fetch_connector; mod tracing_layer; -use client::WorkerBackend; +use client::{FetchHttpExchange, WorkerBackend}; use source_coop_api::api::{CacheTtls, SourceApiClient}; use source_coop_api::resolver::SourceCoopResolver; use source_coop_core::axum::{build_proxy_response, error_response}; use source_coop_core::config::static_file::{StaticConfig, StaticProvider}; +use source_coop_core::oidc_backend::OidcBackendAuth; use source_coop_core::proxy::{ ForwardRequest, HandlerAction, ProxyHandler, RESPONSE_HEADER_ALLOWLIST, }; use source_coop_core::resolver::{DefaultResolver, RequestResolver}; use source_coop_core::sealed_token::TokenKey; +use source_coop_oidc_provider::backend_auth::MaybeOidcAuth; +use source_coop_oidc_provider::jwt::JwtSigner; +use source_coop_oidc_provider::OidcCredentialProvider; use source_coop_sts::{try_handle_sts, try_parse_sts_request, JwksCache}; use axum::body::Body; @@ -78,6 +82,36 @@ async fn fetch( let query = uri.query().map(|q| q.to_string()); let headers = parts.headers; + // Build OIDC backend auth from env secrets/vars. + let (oidc_auth, oidc_discovery) = load_oidc_auth(&env)?; + + // Intercept OIDC discovery endpoints when OIDC provider is configured. + if let Some(disc) = &oidc_discovery { + if path == "/.well-known/openid-configuration" { + let jwks_uri = format!("{}/.well-known/jwks.json", disc.issuer); + let json = source_coop_oidc_provider::discovery::openid_configuration_json( + &disc.issuer, + &jwks_uri, + ); + return Ok(Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(json)) + .unwrap()); + } + if path == "/.well-known/jwks.json" { + let json = source_coop_oidc_provider::jwks::jwks_json( + disc.signer.public_key(), + disc.signer.kid(), + ); + return Ok(Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(json)) + .unwrap()); + } + } + // Intercept STS AssumeRoleWithWebIdentity requests before resolver dispatch. // STS uses STS_CONFIG (falling back to PROXY_CONFIG) for role definitions. if try_parse_sts_request(query.as_deref()).is_some() { @@ -120,7 +154,7 @@ async fn fetch( cache_ttls, ); let resolver = SourceCoopResolver::new(api_client); - let handler = ProxyHandler::new(WorkerBackend, resolver); + let handler = ProxyHandler::new(WorkerBackend, resolver).with_oidc_auth(oidc_auth); return Ok(handle_action( method, @@ -137,7 +171,7 @@ async fn fetch( let config = load_static_config(&env)?; let virtual_host_domain = env.var("VIRTUAL_HOST_DOMAIN").ok().map(|v| v.to_string()); let resolver = DefaultResolver::new(config, virtual_host_domain, token_key); - let handler = ProxyHandler::new(WorkerBackend, resolver); + let handler = ProxyHandler::new(WorkerBackend, resolver).with_oidc_auth(oidc_auth); Ok(handle_action( method, @@ -154,9 +188,9 @@ async fn fetch( // ── Two-phase request handling ────────────────────────────────────── /// Handle the resolved action for any resolver type. -async fn handle_action( +async fn handle_action( method: http::Method, - handler: &ProxyHandler, + handler: &ProxyHandler, client: &reqwest::Client, path: &str, query: Option<&str>, @@ -285,6 +319,45 @@ fn load_sts_config(env: &Env) -> Result { load_config_from_env(env, "STS_CONFIG").or_else(|_| load_config_from_env(env, "PROXY_CONFIG")) } +type OidcAuth = MaybeOidcAuth; + +struct WorkerOidcDiscovery { + issuer: String, + signer: JwtSigner, +} + +/// Load OIDC provider config from env secrets/vars. +/// +/// Returns `MaybeOidcAuth::Enabled` if both `OIDC_PROVIDER_KEY` (secret) and +/// `OIDC_PROVIDER_ISSUER` (var) are set; otherwise `Disabled`. +fn load_oidc_auth(env: &Env) -> Result<(OidcAuth, Option)> { + let key_pem = match env.secret("OIDC_PROVIDER_KEY") { + Ok(val) => Some(val.to_string()), + Err(_) => None, + }; + let issuer = env.var("OIDC_PROVIDER_ISSUER").ok().map(|v| v.to_string()); + + match (key_pem, issuer) { + (Some(pem), Some(issuer)) => { + let signer = JwtSigner::from_pem(&pem, "proxy-key-1".into(), 300) + .map_err(|e| worker::Error::RustError(format!("OIDC signer error: {e}")))?; + let http = FetchHttpExchange; + let provider = OidcCredentialProvider::new( + signer.clone(), + http, + issuer.clone(), + "sts.amazonaws.com".into(), + ); + let auth = MaybeOidcAuth::Enabled( + source_coop_oidc_provider::backend_auth::AwsOidcBackendAuth::new(provider), + ); + let discovery = WorkerOidcDiscovery { issuer, signer }; + Ok((auth, Some(discovery))) + } + _ => Ok((MaybeOidcAuth::Disabled, None)), + } +} + /// Load cache TTL overrides from environment variables. fn load_cache_ttls(env: &Env) -> CacheTtls { let mut cache_ttls = CacheTtls::default(); diff --git a/crates/runtimes/server/Cargo.toml b/crates/runtimes/server/Cargo.toml index 114150b..351d09d 100644 --- a/crates/runtimes/server/Cargo.toml +++ b/crates/runtimes/server/Cargo.toml @@ -8,6 +8,7 @@ description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] source-coop-core = { workspace = true, features = ["axum", "azure", "gcp"] } source-coop-sts.workspace = true +source-coop-oidc-provider.workspace = true source-coop-api.workspace = true axum = { workspace = true, features = ["json", "tokio", "http1", "http2"] } tower-service.workspace = true diff --git a/crates/runtimes/server/README.md b/crates/runtimes/server/README.md index 9f56256..355e500 100644 --- a/crates/runtimes/server/README.md +++ b/crates/runtimes/server/README.md @@ -16,7 +16,7 @@ A `ProxyBackend` implementation plus a server binary: src/ ├── lib.rs Crate root ├── body.rs ProxyResult → Hyper response conversion (Bytes/Empty only) -├── client.rs ServerBackend implementing ProxyBackend +├── client.rs ServerBackend implementing ProxyBackend, ReqwestHttpExchange ├── server.rs Hyper server setup, two-phase request handling, Forward execution └── bin/ └── source-coop-proxy.rs CLI binary entry point @@ -36,6 +36,11 @@ cargo build --release -p source-coop-server --listen 0.0.0.0:9000 \ --domain s3.local +# Enable OIDC backend auth (exchange self-signed JWTs for cloud credentials) +OIDC_PROVIDER_KEY="$(cat private_key.pem)" \ +OIDC_PROVIDER_ISSUER="https://data.example.com" \ +./target/release/source-coop-proxy --config config.toml + # Environment variable for log level RUST_LOG=source_coop=debug ./target/release/source-coop-proxy --config config.toml ``` diff --git a/crates/runtimes/server/src/bin/source-coop-proxy.rs b/crates/runtimes/server/src/bin/source-coop-proxy.rs index 4421c73..5b64f88 100644 --- a/crates/runtimes/server/src/bin/source-coop-proxy.rs +++ b/crates/runtimes/server/src/bin/source-coop-proxy.rs @@ -66,10 +66,15 @@ async fn main() -> Result<(), Box> { .map(|v| TokenKey::from_base64(&v)) .transpose()?; + let oidc_provider_key = std::env::var("OIDC_PROVIDER_KEY").ok(); + let oidc_provider_issuer = std::env::var("OIDC_PROVIDER_ISSUER").ok(); + let server_config = ServerConfig { listen_addr, virtual_host_domain: domain, token_key, + oidc_provider_key, + oidc_provider_issuer, }; run(config, sts_config, server_config).await diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index a3338c6..66bb32b 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -7,6 +7,7 @@ use object_store::ObjectStore; use source_coop_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse}; use source_coop_core::error::ProxyError; use source_coop_core::types::BucketConfig; +use source_coop_oidc_provider::{HttpExchange, OidcProviderError}; use std::sync::Arc; /// Backend for the Tokio/Hyper server runtime. @@ -90,3 +91,35 @@ impl ProxyBackend for ServerBackend { }) } } + +/// [`HttpExchange`] implementation using reqwest (native). +#[derive(Clone)] +pub struct ReqwestHttpExchange { + client: reqwest::Client, +} + +impl ReqwestHttpExchange { + pub fn new(client: reqwest::Client) -> Self { + Self { client } + } +} + +impl HttpExchange for ReqwestHttpExchange { + async fn post_form( + &self, + url: &str, + form: &[(&str, &str)], + ) -> Result { + let resp = self + .client + .post(url) + .form(form) + .send() + .await + .map_err(|e| OidcProviderError::HttpError(e.to_string()))?; + + resp.text() + .await + .map_err(|e| OidcProviderError::HttpError(e.to_string())) + } +} diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 743920b..00d0b85 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -1,6 +1,6 @@ //! HTTP server using axum, wiring everything together. -use crate::client::ServerBackend; +use crate::client::{ReqwestHttpExchange, ServerBackend}; use axum::body::Body; use axum::extract::State; use axum::response::Response; @@ -15,6 +15,9 @@ use source_coop_core::proxy::{ }; use source_coop_core::resolver::DefaultResolver; use source_coop_core::sealed_token::TokenKey; +use source_coop_oidc_provider::backend_auth::MaybeOidcAuth; +use source_coop_oidc_provider::jwt::JwtSigner; +use source_coop_oidc_provider::OidcCredentialProvider; use source_coop_sts::{try_handle_sts, JwksCache}; use std::net::SocketAddr; use std::sync::Arc; @@ -29,6 +32,10 @@ pub struct ServerConfig { pub virtual_host_domain: Option, /// Optional AES-256-GCM key for self-contained encrypted session tokens. pub token_key: Option, + /// PEM-encoded RSA private key for OIDC provider (minting JWTs for backend auth). + pub oidc_provider_key: Option, + /// Issuer URL for the OIDC provider (must be publicly reachable for JWKS discovery). + pub oidc_provider_issuer: Option, } impl Default for ServerConfig { @@ -37,16 +44,27 @@ impl Default for ServerConfig { listen_addr: ([0, 0, 0, 0], 8080).into(), virtual_host_domain: None, token_key: None, + oidc_provider_key: None, + oidc_provider_issuer: None, } } } +type OidcAuth = MaybeOidcAuth; + struct AppState { - handler: ProxyHandler>, + handler: ProxyHandler, OidcAuth>, reqwest_client: reqwest::Client, sts_config: P, jwks_cache: JwksCache, token_key: Option, + /// OIDC discovery data (issuer + signer), set when OIDC provider is configured. + oidc_discovery: Option, +} + +struct OidcDiscovery { + issuer: String, + signer: JwtSigner, } /// Run the S3 proxy server. @@ -82,7 +100,31 @@ where let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); let token_key = server_config.token_key; let resolver = DefaultResolver::new(config, server_config.virtual_host_domain, token_key.clone()); - let handler = ProxyHandler::new(backend, resolver); + + // Build OIDC provider if both key and issuer are configured. + let (oidc_auth, oidc_discovery) = match ( + &server_config.oidc_provider_key, + &server_config.oidc_provider_issuer, + ) { + (Some(key_pem), Some(issuer)) => { + let signer = JwtSigner::from_pem(key_pem, "proxy-key-1".into(), 300) + .map_err(|e| format!("failed to create OIDC JWT signer: {e}"))?; + let http = ReqwestHttpExchange::new(reqwest_client.clone()); + let provider = + OidcCredentialProvider::new(signer.clone(), http, issuer.clone(), "sts.amazonaws.com".into()); + let auth = MaybeOidcAuth::Enabled( + source_coop_oidc_provider::backend_auth::AwsOidcBackendAuth::new(provider), + ); + let discovery = OidcDiscovery { + issuer: issuer.clone(), + signer, + }; + (auth, Some(discovery)) + } + _ => (MaybeOidcAuth::Disabled, None), + }; + + let handler = ProxyHandler::new(backend, resolver).with_oidc_auth(oidc_auth); let state = Arc::new(AppState { handler, @@ -90,6 +132,7 @@ where sts_config, jwks_cache, token_key, + oidc_discovery, }); let app = Router::new() @@ -120,6 +163,31 @@ async fn request_handler( "incoming request" ); + // Intercept OIDC discovery endpoints when OIDC provider is configured. + if let Some(disc) = &state.oidc_discovery { + if path == "/.well-known/openid-configuration" { + let jwks_uri = format!("{}/.well-known/jwks.json", disc.issuer); + let json = + source_coop_oidc_provider::discovery::openid_configuration_json(&disc.issuer, &jwks_uri); + return Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(json)) + .unwrap(); + } + if path == "/.well-known/jwks.json" { + let json = source_coop_oidc_provider::jwks::jwks_json( + disc.signer.public_key(), + disc.signer.kid(), + ); + return Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(json)) + .unwrap(); + } + } + // Intercept STS AssumeRoleWithWebIdentity requests if let Some((status, xml)) = try_handle_sts(query.as_deref(), &state.sts_config, &state.jwks_cache, state.token_key.as_ref()).await From 17d96b877c6f75144ac7521ce2a5dfc47aa7063c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Fri, 27 Feb 2026 15:55:52 -0500 Subject: [PATCH 54/82] ci: add audit check --- .github/workflows/ci.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f4b4471..d8c0692 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,6 +53,15 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo test + audit: + name: Audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: rustsec/audit-check@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + build: name: Build runs-on: ubuntu-latest From abe26b187f1ef35bd4b00e9a01b276704d388510 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 07:42:27 -0500 Subject: [PATCH 55/82] chore: cargo fmt --- crates/cli/src/main.rs | 3 +- crates/libs/core/src/auth.rs | 42 +++++++------------ crates/libs/core/src/config/dynamodb.rs | 1 - crates/libs/core/src/config/http.rs | 1 - crates/libs/core/src/config/postgres.rs | 1 - crates/libs/core/src/config/static_file.rs | 1 - crates/libs/core/src/oidc_backend.rs | 5 +-- crates/libs/core/src/sealed_token.rs | 13 ++++-- crates/libs/oidc-provider/src/backend_auth.rs | 33 +++++++-------- crates/runtimes/server/src/server.rs | 26 ++++++++---- 10 files changed, 59 insertions(+), 67 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d5ef051..2f271ce 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -117,8 +117,7 @@ async fn run_login(args: LoginArgs) -> Result<(), String> { // 3. STS credential exchange eprintln!("Exchanging token for credentials..."); - let creds = - sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; + let creds = sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; // 4. Cache credentials let path = cache::write_credentials(&args.role_arn, &creds)?; diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index c75770f..0d4b647 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -220,10 +220,7 @@ pub async fn resolve_identity( match key.unseal(session_token)? { Some(creds) => { - if !constant_time_eq( - sig.access_key_id.as_bytes(), - creds.access_key_id.as_bytes(), - ) { + if !constant_time_eq(sig.access_key_id.as_bytes(), creds.access_key_id.as_bytes()) { tracing::warn!( header_key = %sig.access_key_id, sealed_key = %creds.access_key_id, @@ -248,9 +245,7 @@ pub async fn resolve_identity( scopes = ?creds.allowed_scopes, "sealed token identity resolved" ); - return Ok(ResolvedIdentity::Temporary { - credentials: creds, - }); + return Ok(ResolvedIdentity::Temporary { credentials: creds }); } None => { tracing::warn!("session token could not be unsealed (decryption failed)"); @@ -626,10 +621,8 @@ mod tests { run(async { let key_bytes = [0x42u8; 32]; - let encoded = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - key_bytes, - ); + let encoded = + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key_bytes); let token_key = TokenKey::from_base64(&encoded).unwrap(); let config = MockConfig::empty(); @@ -688,10 +681,8 @@ mod tests { run(async { let key_bytes = [0x42u8; 32]; - let encoded = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - key_bytes, - ); + let encoded = + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key_bytes); let token_key = TokenKey::from_base64(&encoded).unwrap(); let real_secret = "TempSecretKey1234567890EXAMPLE000000000000"; @@ -911,14 +902,8 @@ mod tests { headers.insert("authorization", auth.parse().unwrap()); // Verify with UNSORTED query (as it arrives from the raw URL) - let sig = parse_sigv4_auth( - headers - .get("authorization") - .unwrap() - .to_str() - .unwrap(), - ) - .unwrap(); + let sig = + parse_sigv4_auth(headers.get("authorization").unwrap().to_str().unwrap()).unwrap(); let result = verify_sigv4_signature( &http::Method::GET, @@ -931,7 +916,10 @@ mod tests { ) .unwrap(); - assert!(result, "S3 ListObjects with security token and host:port must verify"); + assert!( + result, + "S3 ListObjects with security token and host:port must verify" + ); } // ── Sealed token tests ────────────────────────────────────────── @@ -943,10 +931,8 @@ mod tests { run(async { let key_bytes = [0x42u8; 32]; - let encoded = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - key_bytes, - ); + let encoded = + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key_bytes); let token_key = TokenKey::from_base64(&encoded).unwrap(); let secret = "TempSecretKey1234567890EXAMPLE000000000000"; diff --git a/crates/libs/core/src/config/dynamodb.rs b/crates/libs/core/src/config/dynamodb.rs index d8fd454..d5723bb 100644 --- a/crates/libs/core/src/config/dynamodb.rs +++ b/crates/libs/core/src/config/dynamodb.rs @@ -166,5 +166,4 @@ impl ConfigProvider for DynamoDbProvider { None => Ok(None), } } - } diff --git a/crates/libs/core/src/config/http.rs b/crates/libs/core/src/config/http.rs index ab2e5c9..4faa84c 100644 --- a/crates/libs/core/src/config/http.rs +++ b/crates/libs/core/src/config/http.rs @@ -155,7 +155,6 @@ impl ConfigProvider for HttpProvider { .map(Some) .map_err(|e| ProxyError::ConfigError(e.to_string())) } - } #[cfg(test)] diff --git a/crates/libs/core/src/config/postgres.rs b/crates/libs/core/src/config/postgres.rs index 6e90092..2b4f673 100644 --- a/crates/libs/core/src/config/postgres.rs +++ b/crates/libs/core/src/config/postgres.rs @@ -113,5 +113,4 @@ impl ConfigProvider for PostgresProvider { }) .transpose() } - } diff --git a/crates/libs/core/src/config/static_file.rs b/crates/libs/core/src/config/static_file.rs index 0b99058..623e420 100644 --- a/crates/libs/core/src/config/static_file.rs +++ b/crates/libs/core/src/config/static_file.rs @@ -119,5 +119,4 @@ impl ConfigProvider for StaticProvider { .find(|c| c.access_key_id == access_key_id) .cloned()) } - } diff --git a/crates/libs/core/src/oidc_backend.rs b/crates/libs/core/src/oidc_backend.rs index d9d0f76..031c4d2 100644 --- a/crates/libs/core/src/oidc_backend.rs +++ b/crates/libs/core/src/oidc_backend.rs @@ -33,10 +33,7 @@ pub trait OidcBackendAuth: MaybeSend + 'static { pub struct NoOidcAuth; impl OidcBackendAuth for NoOidcAuth { - async fn resolve_credentials( - &self, - config: &BucketConfig, - ) -> Result { + async fn resolve_credentials(&self, config: &BucketConfig) -> Result { if config.option("auth_type") == Some("oidc") { return Err(ProxyError::ConfigError( "bucket requires auth_type=oidc but no OIDC provider is configured".into(), diff --git a/crates/libs/core/src/sealed_token.rs b/crates/libs/core/src/sealed_token.rs index 6ecf509..c82b0f3 100644 --- a/crates/libs/core/src/sealed_token.rs +++ b/crates/libs/core/src/sealed_token.rs @@ -24,7 +24,9 @@ impl TokenKey { pub fn from_base64(encoded: &str) -> Result { let bytes = base64::engine::general_purpose::STANDARD .decode(encoded.trim()) - .map_err(|e| ProxyError::ConfigError(format!("invalid SESSION_TOKEN_KEY base64: {e}")))?; + .map_err(|e| { + ProxyError::ConfigError(format!("invalid SESSION_TOKEN_KEY base64: {e}")) + })?; if bytes.len() != 32 { return Err(ProxyError::ConfigError(format!( "SESSION_TOKEN_KEY must be 32 bytes, got {}", @@ -40,8 +42,8 @@ impl TokenKey { /// /// Format: `base64url(nonce[12] || ciphertext+tag)` pub fn seal(&self, creds: &TemporaryCredentials) -> Result { - let plaintext = - serde_json::to_vec(creds).map_err(|e| ProxyError::Internal(format!("seal json: {e}")))?; + let plaintext = serde_json::to_vec(creds) + .map_err(|e| ProxyError::Internal(format!("seal json: {e}")))?; let nonce = Aes256Gcm::generate_nonce(&mut OsRng); let ciphertext = self .0 @@ -144,7 +146,10 @@ mod tests { #[test] fn non_sealed_token_returns_none() { let key = make_key(); - assert!(key.unseal("FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng").unwrap().is_none()); + assert!(key + .unseal("FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng") + .unwrap() + .is_none()); } #[test] diff --git a/crates/libs/oidc-provider/src/backend_auth.rs b/crates/libs/oidc-provider/src/backend_auth.rs index 19f944c..41af6be 100644 --- a/crates/libs/oidc-provider/src/backend_auth.rs +++ b/crates/libs/oidc-provider/src/backend_auth.rs @@ -23,16 +23,13 @@ impl AwsOidcBackendAuth { Self { provider } } - async fn resolve_aws( - &self, - config: &BucketConfig, - ) -> Result { + async fn resolve_aws(&self, config: &BucketConfig) -> Result { let role_arn = config.option("oidc_role_arn").ok_or_else(|| { - ProxyError::ConfigError("auth_type=oidc requires 'oidc_role_arn' in backend_options".into()) + ProxyError::ConfigError( + "auth_type=oidc requires 'oidc_role_arn' in backend_options".into(), + ) })?; - let subject = config - .option("oidc_subject") - .unwrap_or("s3-proxy"); + let subject = config.option("oidc_subject").unwrap_or("s3-proxy"); let exchange = AwsExchange::new(role_arn.to_string()); let creds = self @@ -61,10 +58,7 @@ impl AwsOidcBackendAuth { } impl OidcBackendAuth for AwsOidcBackendAuth { - async fn resolve_credentials( - &self, - config: &BucketConfig, - ) -> Result { + async fn resolve_credentials(&self, config: &BucketConfig) -> Result { if config.option("auth_type") != Some("oidc") { return Ok(config.clone()); } @@ -90,10 +84,7 @@ pub enum MaybeOidcAuth { } impl OidcBackendAuth for MaybeOidcAuth { - async fn resolve_credentials( - &self, - config: &BucketConfig, - ) -> Result { + async fn resolve_credentials(&self, config: &BucketConfig) -> Result { match self { MaybeOidcAuth::Enabled(auth) => auth.resolve_credentials(config).await, MaybeOidcAuth::Disabled => { @@ -167,7 +158,10 @@ mod tests { let mut opts = HashMap::new(); opts.insert("auth_type".into(), "oidc".into()); opts.insert("oidc_role_arn".into(), "arn:aws:iam::123:role/Test".into()); - opts.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + opts.insert( + "endpoint".into(), + "https://s3.us-east-1.amazonaws.com".into(), + ); opts.insert("bucket_name".into(), "my-bucket".into()); opts.insert("region".into(), "us-east-1".into()); BucketConfig { @@ -184,7 +178,10 @@ mod tests { let mut opts = HashMap::new(); opts.insert("access_key_id".into(), "AKID_STATIC".into()); opts.insert("secret_access_key".into(), "secret_static".into()); - opts.insert("endpoint".into(), "https://s3.us-east-1.amazonaws.com".into()); + opts.insert( + "endpoint".into(), + "https://s3.us-east-1.amazonaws.com".into(), + ); opts.insert("bucket_name".into(), "my-bucket".into()); BucketConfig { name: "test".into(), diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 00d0b85..8bbba01 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -99,7 +99,8 @@ where let reqwest_client = backend.client().clone(); let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); let token_key = server_config.token_key; - let resolver = DefaultResolver::new(config, server_config.virtual_host_domain, token_key.clone()); + let resolver = + DefaultResolver::new(config, server_config.virtual_host_domain, token_key.clone()); // Build OIDC provider if both key and issuer are configured. let (oidc_auth, oidc_discovery) = match ( @@ -110,8 +111,12 @@ where let signer = JwtSigner::from_pem(key_pem, "proxy-key-1".into(), 300) .map_err(|e| format!("failed to create OIDC JWT signer: {e}"))?; let http = ReqwestHttpExchange::new(reqwest_client.clone()); - let provider = - OidcCredentialProvider::new(signer.clone(), http, issuer.clone(), "sts.amazonaws.com".into()); + let provider = OidcCredentialProvider::new( + signer.clone(), + http, + issuer.clone(), + "sts.amazonaws.com".into(), + ); let auth = MaybeOidcAuth::Enabled( source_coop_oidc_provider::backend_auth::AwsOidcBackendAuth::new(provider), ); @@ -167,8 +172,10 @@ async fn request_handler( if let Some(disc) = &state.oidc_discovery { if path == "/.well-known/openid-configuration" { let jwks_uri = format!("{}/.well-known/jwks.json", disc.issuer); - let json = - source_coop_oidc_provider::discovery::openid_configuration_json(&disc.issuer, &jwks_uri); + let json = source_coop_oidc_provider::discovery::openid_configuration_json( + &disc.issuer, + &jwks_uri, + ); return Response::builder() .status(200) .header("content-type", "application/json") @@ -189,8 +196,13 @@ async fn request_handler( } // Intercept STS AssumeRoleWithWebIdentity requests - if let Some((status, xml)) = - try_handle_sts(query.as_deref(), &state.sts_config, &state.jwks_cache, state.token_key.as_ref()).await + if let Some((status, xml)) = try_handle_sts( + query.as_deref(), + &state.sts_config, + &state.jwks_cache, + state.token_key.as_ref(), + ) + .await { return Response::builder() .status(status) From dde94e73b834fdde4746b856108a7231ea3d5649 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 07:45:05 -0500 Subject: [PATCH 56/82] chore: clippy --- crates/libs/oidc-provider/src/backend_auth.rs | 2 +- crates/runtimes/server/src/server.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/libs/oidc-provider/src/backend_auth.rs b/crates/libs/oidc-provider/src/backend_auth.rs index 41af6be..6a747d3 100644 --- a/crates/libs/oidc-provider/src/backend_auth.rs +++ b/crates/libs/oidc-provider/src/backend_auth.rs @@ -79,7 +79,7 @@ impl OidcBackendAuth for AwsOidcBackendAuth { /// When disabled and a bucket specifies `auth_type=oidc`, a `ConfigError` /// is returned (same as `NoOidcAuth`). pub enum MaybeOidcAuth { - Enabled(AwsOidcBackendAuth), + Enabled(Box>), Disabled, } diff --git a/crates/runtimes/server/src/server.rs b/crates/runtimes/server/src/server.rs index 8bbba01..7ff2c60 100644 --- a/crates/runtimes/server/src/server.rs +++ b/crates/runtimes/server/src/server.rs @@ -117,9 +117,9 @@ where issuer.clone(), "sts.amazonaws.com".into(), ); - let auth = MaybeOidcAuth::Enabled( + let auth = MaybeOidcAuth::Enabled(Box::new( source_coop_oidc_provider::backend_auth::AwsOidcBackendAuth::new(provider), - ); + )); let discovery = OidcDiscovery { issuer: issuer.clone(), signer, From 16cd5e716730ad30294ec99b58b3c283921c0e14 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 07:55:58 -0500 Subject: [PATCH 57/82] ci: cargo audit --- .cargo/audit.toml | 3 +++ Cargo.lock | 52 +++++++++++++++++++++++++++++++++++------------ Cargo.toml | 2 +- 3 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 .cargo/audit.toml diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..1e2b83f --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,3 @@ +[advisories] +# Marvin Attack: timing side-channel in rsa crate. No fix available upstream. +ignore = ["RUSTSEC-2023-0071"] diff --git a/Cargo.lock b/Cargo.lock index 045c901..4b588b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1771,6 +1771,41 @@ name = "object_store" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http 1.4.0", + "http-body-util", + "humantime", + "hyper 1.8.1", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.38.4", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2858065e55c148d294a9f3aae3b0fa9458edadb41a108397094566f4e3c0dfb" dependencies = [ "async-trait", "base64", @@ -1791,7 +1826,7 @@ dependencies = [ "rand 0.9.2", "reqwest", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2346,15 +2381,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -2660,7 +2686,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "js-sys", - "object_store", + "object_store 0.12.5", "quick-xml 0.37.5", "reqwest", "serde", @@ -2712,7 +2738,7 @@ dependencies = [ "hex", "hmac", "http 1.4.0", - "object_store", + "object_store 0.13.1", "quick-xml 0.37.5", "reqwest", "serde", @@ -2755,7 +2781,7 @@ dependencies = [ "futures", "http 1.4.0", "http-body-util", - "object_store", + "object_store 0.13.1", "reqwest", "serde", "source-coop-api", diff --git a/Cargo.toml b/Cargo.toml index 72df8a2..804380e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ rsa = "0.9" aes-gcm = "0.10" # Object store -object_store = { version = "0.12", default-features = false, features = ["aws"] } +object_store = { version = "0.13.1", default-features = false, features = ["aws"] } futures = "0.3" # XML From 627afdb9656486b31dfd7f8a9cfd8e3bf2806a6f Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 08:00:41 -0500 Subject: [PATCH 58/82] chore: fix subcrate dependencies --- Cargo.lock | 41 ++------------------------- crates/runtimes/cf-workers/Cargo.toml | 2 +- crates/runtimes/cf-workers/src/lib.rs | 4 +-- 3 files changed, 6 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b588b9..5f54ea5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1766,41 +1766,6 @@ dependencies = [ "libm", ] -[[package]] -name = "object_store" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" -dependencies = [ - "async-trait", - "base64", - "bytes", - "chrono", - "form_urlencoded", - "futures", - "http 1.4.0", - "http-body-util", - "humantime", - "hyper 1.8.1", - "itertools", - "md-5", - "parking_lot", - "percent-encoding", - "quick-xml 0.38.4", - "rand 0.9.2", - "reqwest", - "ring", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror", - "tokio", - "tracing", - "url", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "object_store" version = "0.13.1" @@ -2686,7 +2651,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "js-sys", - "object_store 0.12.5", + "object_store", "quick-xml 0.37.5", "reqwest", "serde", @@ -2738,7 +2703,7 @@ dependencies = [ "hex", "hmac", "http 1.4.0", - "object_store 0.13.1", + "object_store", "quick-xml 0.37.5", "reqwest", "serde", @@ -2781,7 +2746,7 @@ dependencies = [ "futures", "http 1.4.0", "http-body-util", - "object_store 0.13.1", + "object_store", "reqwest", "serde", "source-coop-api", diff --git a/crates/runtimes/cf-workers/Cargo.toml b/crates/runtimes/cf-workers/Cargo.toml index a8f9f2f..c7d1a2b 100644 --- a/crates/runtimes/cf-workers/Cargo.toml +++ b/crates/runtimes/cf-workers/Cargo.toml @@ -23,7 +23,7 @@ thiserror.workspace = true chrono.workspace = true quick-xml.workspace = true url.workspace = true -object_store = { version = "0.12", default-features = false, features = ["aws"] } +object_store = { workspace = true } futures.workspace = true http-body.workspace = true http-body-util.workspace = true diff --git a/crates/runtimes/cf-workers/src/lib.rs b/crates/runtimes/cf-workers/src/lib.rs index 828b49d..4f594bc 100644 --- a/crates/runtimes/cf-workers/src/lib.rs +++ b/crates/runtimes/cf-workers/src/lib.rs @@ -348,9 +348,9 @@ fn load_oidc_auth(env: &Env) -> Result<(OidcAuth, Option)> issuer.clone(), "sts.amazonaws.com".into(), ); - let auth = MaybeOidcAuth::Enabled( + let auth = MaybeOidcAuth::Enabled(Box::new( source_coop_oidc_provider::backend_auth::AwsOidcBackendAuth::new(provider), - ); + )); let discovery = WorkerOidcDiscovery { issuer, signer }; Ok((auth, Some(discovery))) } From ab2a5975129637bf6c79cdbf085a626252514926 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 08:08:34 -0500 Subject: [PATCH 59/82] chore: integrate ci checks in git hooks --- .githooks/pre-commit | 3 +++ Makefile | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100755 .githooks/pre-commit diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..b6dfb0e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -e +make ci diff --git a/Makefile b/Makefile index 088fb83..d697b49 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: check test run\:server run\:workers +.PHONY: check test run\:server run\:workers ci setup check: cargo check @@ -28,3 +28,8 @@ build\:cli: build\:cli\:staging: cargo build -p source-coop-cli --no-default-features --features staging + +ci: fmt clippy check test + +setup: + git config core.hooksPath .githooks From de9f60797b2291c5c72309a61b7026d4f6404e31 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 16:05:30 -0500 Subject: [PATCH 60/82] chore: fixup makefile --- .githooks/pre-commit | 2 +- Makefile | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.githooks/pre-commit b/.githooks/pre-commit index b6dfb0e..10d7119 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,3 +1,3 @@ #!/usr/bin/env bash set -e -make ci +make ci-fast diff --git a/Makefile b/Makefile index d697b49..b2059f6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: check test run\:server run\:workers ci setup +.PHONY: check test run-server run-workers ci setup check: cargo check @@ -6,30 +6,31 @@ check: fmt: cargo fmt -- --check -fmt\:fix: +fmt-fix: cargo fmt clippy: cargo clippy -- -D warnings -clippy\:fix: +clippy-fix: cargo clippy --fix --allow-dirty --allow-staged test: cargo test -run\:server: +run-server: cargo run -p source-coop-server -- $(ARGS) -run\:workers: +run-workers: npx wrangler dev --cwd crates/runtimes/cf-workers -build\:cli: +build-cli: cargo build -p source-coop-cli -build\:cli\:staging: +build-cli-staging: cargo build -p source-coop-cli --no-default-features --features staging -ci: fmt clippy check test +ci-fast: fmt clippy check +ci: ci-fast test setup: git config core.hooksPath .githooks From 2fc733957af10a049b3ee152ce926b2cf870f24f Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 16:16:47 -0500 Subject: [PATCH 61/82] ci: utilize caching --- .github/workflows/ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d8c0692..66757ad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,9 +58,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: rustsec/audit-check@v2 + - uses: dtolnay/rust-toolchain@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} + toolchain: stable + - uses: Swatinem/rust-cache@v2 + - run: cargo install cargo-audit + - run: cargo audit build: name: Build From 461e264258ade1534215ee84c9efbb2ef6447b75 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 16:23:15 -0500 Subject: [PATCH 62/82] chore: speed up ci-fast --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b2059f6..c9a74a0 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ check: cargo check + +check-wasm: cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown fmt: @@ -29,7 +31,7 @@ build-cli: build-cli-staging: cargo build -p source-coop-cli --no-default-features --features staging -ci-fast: fmt clippy check +ci-fast: fmt clippy check-wasm ci: ci-fast test setup: From 59a217fe24aa4d1d67b8a2676d9bb33332b4fd62 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 22:18:27 -0500 Subject: [PATCH 63/82] chore: rm CLI CLI now lives at https://github.com/source-cooperative/source-coop-cli --- .github/workflows/ci.yaml | 2 - Cargo.lock | 213 ----------------------------------- Cargo.toml | 2 - crates/cli/Cargo.toml | 30 ----- crates/cli/README.md | 114 ------------------- crates/cli/src/cache.rs | 159 -------------------------- crates/cli/src/main.rs | 147 ------------------------ crates/cli/src/oidc.rs | 230 -------------------------------------- crates/cli/src/output.rs | 20 ---- crates/cli/src/sts.rs | 104 ----------------- 10 files changed, 1021 deletions(-) delete mode 100644 crates/cli/Cargo.toml delete mode 100644 crates/cli/README.md delete mode 100644 crates/cli/src/cache.rs delete mode 100644 crates/cli/src/main.rs delete mode 100644 crates/cli/src/oidc.rs delete mode 100644 crates/cli/src/output.rs delete mode 100644 crates/cli/src/sts.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 66757ad..0f59ef3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,5 +76,3 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build server run: cargo build -p source-coop-server - - name: Build CLI - run: cargo build -p source-coop-cli diff --git a/Cargo.lock b/Cargo.lock index 5f54ea5..6b5b8ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,56 +61,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "0.6.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.101" @@ -580,46 +530,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "4.5.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" - [[package]] name = "cmake" version = "0.1.57" @@ -629,12 +539,6 @@ dependencies = [ "cc", ] -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -767,27 +671,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -1516,31 +1399,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is-docker" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" -dependencies = [ - "once_cell", -] - -[[package]] -name = "is-wsl" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" -dependencies = [ - "is-docker", - "once_cell", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" version = "0.14.0" @@ -1809,41 +1667,18 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "opaque-debug" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "open" -version = "5.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb73a7fa3799b198970490a51174027ba0d4ec504b03cd08caf513d40024bc" -dependencies = [ - "is-wsl", - "libc", - "pathdiff", -] - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "outref" version = "0.5.2" @@ -1879,12 +1714,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -2181,17 +2010,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror", -] - [[package]] name = "regex-automata" version = "0.4.14" @@ -2669,25 +2487,6 @@ dependencies = [ "worker", ] -[[package]] -name = "source-coop-cli" -version = "1.0.4" -dependencies = [ - "base64", - "chrono", - "clap", - "dirs", - "open", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", - "serde", - "serde_json", - "sha2", - "tokio", - "url", -] - [[package]] name = "source-coop-core" version = "1.0.4" @@ -3011,12 +2810,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subtle" version = "2.6.1" @@ -3437,12 +3230,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.21.0" diff --git a/Cargo.toml b/Cargo.toml index 804380e..df32245 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ members = [ "crates/libs/api", "crates/runtimes/server", "crates/runtimes/cf-workers", - "crates/cli", ] # Worker crate is excluded from default builds because it contains !Send # WASM types that only compile correctly for wasm32 targets. Build it @@ -17,7 +16,6 @@ default-members = [ "crates/libs/oidc-provider", "crates/libs/api", "crates/runtimes/server", - "crates/cli", ] resolver = "2" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml deleted file mode 100644 index ed6f81e..0000000 --- a/crates/cli/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "source-coop-cli" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "CLI for Source Cooperative — OIDC login and STS credential exchange" - -[[bin]] -name = "source-coop" -path = "src/main.rs" - -[dependencies] -clap = { version = "4", features = ["derive", "env"] } -open = "5" -reqwest = { workspace = true } -tokio = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -url = { workspace = true } -sha2 = { workspace = true } -base64 = { workspace = true } -rand = "0.8" -quick-xml = { workspace = true } -chrono = { workspace = true } -dirs = "6" - -[features] -default = ["production"] -production = [] -staging = [] diff --git a/crates/cli/README.md b/crates/cli/README.md deleted file mode 100644 index 13e6f3c..0000000 --- a/crates/cli/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# source-coop CLI - -Authenticate with the Source Cooperative data proxy and obtain temporary S3 credentials. - -Uses the OAuth2 Authorization Code flow with PKCE to authenticate via browser, then exchanges the OIDC ID token at the proxy's STS endpoint for temporary AWS credentials. - -## Install - -```bash -cargo install --path crates/cli -``` - -## Usage - -### Recommended: login + credential-process - -1. Log in once (opens browser, caches credentials to `~/.source-coop/credentials/`): - -```bash -source-coop login -``` - -2. Configure `~/.aws/config` to use cached credentials: - -```ini -[profile source-coop] -credential_process = source-coop credential-process -endpoint_url = https://data.source.coop -``` - -3. Use AWS tools normally: - -```bash -aws s3 ls s3://my-bucket/ --profile source-coop -``` - -When credentials expire, run `source-coop login` again. - -### Multiple roles - -Each role's credentials are cached separately: - -```bash -source-coop login --role-arn reader-role -source-coop login --role-arn admin-role -``` - -```ini -[profile source-coop] -credential_process = source-coop credential-process --role-arn reader-role -endpoint_url = https://data.source.coop - -[profile source-coop-admin] -credential_process = source-coop credential-process --role-arn admin-role -endpoint_url = https://data.source.coop -``` - -### Login options - -| Flag | Env var | Default | Description | -|------|---------|---------|-------------| -| `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | -| `--client-id` | `SOURCE_OIDC_CLIENT_ID` | `d037d00b-...` | OAuth2 client ID | -| `--proxy-url` | `SOURCE_PROXY_URL` | `https://data.source.coop` | S3 proxy URL for STS | -| `--role-arn` | `SOURCE_ROLE_ARN` | `source-coop-user` | Role ARN to assume | -| `--format` | | `credential-process` | Output format: `credential-process` or `env` | -| `--duration` | | | Session duration in seconds | -| `--scope` | | `openid` | OAuth2 scopes | -| `--port` | | `0` (random) | Local callback port | - -### Output formats - -In addition to caching, `login` prints credentials to stdout: - -**credential-process** (default) — AWS credential_process JSON: - -```bash -source-coop login -``` - -**env** — shell export statements: - -```bash -eval $(source-coop login --format env) -``` - -## OIDC provider setup - -The CLI uses the OAuth2 Authorization Code flow with PKCE. It starts a temporary local server on `http://127.0.0.1:{port}/callback` to receive the authorization code redirect. - -The OAuth2 client must have a matching redirect URI registered. There are two approaches: - -### Option A: Allow any port (recommended) - -Register `http://127.0.0.1/callback` as a redirect URI on the OAuth2 client. Per [RFC 8252 Section 7.3](https://datatracker.ietf.org/doc/html/rfc8252#section-7.3), loopback redirect URIs should allow any port. Ory Network follows this convention — registering the base URI without a port permits any port. - -The CLI defaults to `--port 0` (OS-assigned random available port), which works with this setup. - -### Option B: Fixed port - -Register a specific redirect URI (e.g. `http://127.0.0.1:8400/callback`) and run the CLI with the matching port: - -```bash -source-coop login --role-arn --port 8400 -``` - -### Client configuration - -The OAuth2 client should be configured as a **public client** (no client secret) with: - -- **Grant type**: Authorization Code -- **Token endpoint auth method**: `none` (public client, PKCE used instead) -- **Allowed scopes**: `openid` -- **Redirect URIs**: `http://127.0.0.1/callback` (see above) diff --git a/crates/cli/src/cache.rs b/crates/cli/src/cache.rs deleted file mode 100644 index d70f18d..0000000 --- a/crates/cli/src/cache.rs +++ /dev/null @@ -1,159 +0,0 @@ -use crate::sts::Credentials; -use chrono::Utc; -use std::fs; -use std::io; -use std::path::PathBuf; - -/// Replace any character that isn't alphanumeric, `-`, or `_` with `_`. -fn sanitize_role_arn(role_arn: &str) -> String { - role_arn - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || c == '-' || c == '_' { - c - } else { - '_' - } - }) - .collect() -} - -/// Full path to the credentials cache file for a given role. -fn cache_path(role_arn: &str) -> Result { - let home = dirs::home_dir().ok_or("Could not determine home directory")?; - let sanitized = sanitize_role_arn(role_arn); - Ok(home - .join(".source-coop") - .join("credentials") - .join(format!("{sanitized}.json"))) -} - -/// Write credentials to the per-role cache file. -/// Creates `~/.source-coop/credentials/` if it does not exist. -/// Sets file permissions to 0600 on Unix. -pub fn write_credentials(role_arn: &str, creds: &Credentials) -> Result { - let path = cache_path(role_arn)?; - let dir = path.parent().unwrap(); - - fs::create_dir_all(dir) - .map_err(|e| format!("Failed to create cache directory {}: {e}", dir.display()))?; - - let json = serde_json::to_string_pretty(creds) - .map_err(|e| format!("Failed to serialize credentials: {e}"))?; - - fs::write(&path, &json) - .map_err(|e| format!("Failed to write credentials cache {}: {e}", path.display()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&path, fs::Permissions::from_mode(0o600)) - .map_err(|e| format!("Failed to set permissions on {}: {e}", path.display()))?; - } - - Ok(path) -} - -/// Read credentials from the cache file for a given role. -/// Returns `None` if the file does not exist. -pub fn read_credentials(role_arn: &str) -> Result, String> { - let path = cache_path(role_arn)?; - match fs::read_to_string(&path) { - Ok(contents) => { - let creds: Credentials = serde_json::from_str(&contents) - .map_err(|e| format!("Failed to parse credentials cache: {e}"))?; - Ok(Some(creds)) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(format!( - "Failed to read credentials cache {}: {e}", - path.display() - )), - } -} - -/// Check if credentials are expired or will expire within a 60-second buffer. -pub fn is_expired(creds: &Credentials) -> Result { - let expiration = chrono::DateTime::parse_from_rfc3339(&creds.expiration).map_err(|e| { - format!( - "Failed to parse expiration timestamp '{}': {e}", - creds.expiration - ) - })?; - - let now = Utc::now(); - let buffer = chrono::Duration::seconds(60); - - Ok(expiration <= now + buffer) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sample_creds(expiration: &str) -> Credentials { - Credentials { - access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(), - session_token: "FwoGZXIvYXdzEtest".to_string(), - expiration: expiration.to_string(), - } - } - - #[test] - fn sanitize_simple_name() { - assert_eq!(sanitize_role_arn("source-coop-user"), "source-coop-user"); - } - - #[test] - fn sanitize_arn_with_special_chars() { - assert_eq!( - sanitize_role_arn("arn:aws:iam::123:role/Foo"), - "arn_aws_iam__123_role_Foo" - ); - } - - #[test] - fn sanitize_preserves_underscores() { - assert_eq!(sanitize_role_arn("my_role-name"), "my_role-name"); - } - - #[test] - fn expired_future_date() { - let future = (Utc::now() + chrono::Duration::hours(1)).to_rfc3339(); - let creds = sample_creds(&future); - assert!(!is_expired(&creds).unwrap()); - } - - #[test] - fn expired_past_date() { - let past = (Utc::now() - chrono::Duration::hours(1)).to_rfc3339(); - let creds = sample_creds(&past); - assert!(is_expired(&creds).unwrap()); - } - - #[test] - fn expired_within_buffer() { - // 30 seconds from now is within the 60s buffer - let near_future = (Utc::now() + chrono::Duration::seconds(30)).to_rfc3339(); - let creds = sample_creds(&near_future); - assert!(is_expired(&creds).unwrap()); - } - - #[test] - fn expired_invalid_timestamp() { - let creds = sample_creds("not-a-timestamp"); - assert!(is_expired(&creds).is_err()); - } - - #[test] - fn round_trip_serialization() { - let creds = sample_creds("2026-03-01T00:00:00Z"); - let json = serde_json::to_string_pretty(&creds).unwrap(); - let loaded: Credentials = serde_json::from_str(&json).unwrap(); - assert_eq!(loaded.access_key_id, creds.access_key_id); - assert_eq!(loaded.secret_access_key, creds.secret_access_key); - assert_eq!(loaded.session_token, creds.session_token); - assert_eq!(loaded.expiration, creds.expiration); - } -} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs deleted file mode 100644 index 2f271ce..0000000 --- a/crates/cli/src/main.rs +++ /dev/null @@ -1,147 +0,0 @@ -mod cache; -mod oidc; -mod output; -mod sts; - -use clap::{Parser, Subcommand, ValueEnum}; - -#[cfg(feature = "staging")] -mod defaults { - pub const ISSUER: &str = "https://auth.staging.source.coop"; - pub const CLIENT_ID: &str = "c445cc61-9884-44a8-b051-8d8f7273ffc1"; - pub const PROXY_URL: &str = "https://staging.data.source.coop"; - pub const ROLE_ARN: &str = "source-coop-user"; -} - -#[cfg(not(feature = "staging"))] -mod defaults { - pub const ISSUER: &str = "https://auth.source.coop"; - pub const CLIENT_ID: &str = "d037d00b-09c7-4815-ac39-2a0b9fae40c6"; - pub const PROXY_URL: &str = "https://data.source.coop"; - pub const ROLE_ARN: &str = "source-coop-user"; -} - -#[derive(Parser)] -#[command(name = "source-coop", about = "Source Cooperative CLI")] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Authenticate via OIDC and obtain temporary S3 credentials - Login(LoginArgs), - /// Output cached credentials for AWS credential_process - CredentialProcess(CredentialProcessArgs), -} - -#[derive(Parser)] -struct LoginArgs { - /// OIDC issuer URL - #[arg(long, env = "SOURCE_OIDC_ISSUER", default_value = defaults::ISSUER)] - issuer: String, - - /// OAuth2 client ID - #[arg(long, env = "SOURCE_OIDC_CLIENT_ID", default_value = defaults::CLIENT_ID)] - client_id: String, - - /// S3 proxy URL for STS - #[arg(long, env = "SOURCE_PROXY_URL", default_value = defaults::PROXY_URL)] - proxy_url: String, - - /// Role ARN to assume - #[arg(long, env = "SOURCE_ROLE_ARN", default_value = defaults::ROLE_ARN)] - role_arn: String, - - /// Output format - #[arg(long, default_value = "credential-process")] - format: OutputFormat, - - /// Session duration in seconds - #[arg(long)] - duration: Option, - - /// OAuth2 scopes - #[arg(long, default_value = "openid")] - scope: String, - - /// Local callback port (0 for random available port) - #[arg(long, default_value = "0")] - port: u16, -} - -#[derive(Parser)] -struct CredentialProcessArgs { - /// Role ARN to read cached credentials for - #[arg(long, env = "SOURCE_ROLE_ARN", default_value = defaults::ROLE_ARN)] - role_arn: String, -} - -#[derive(Clone, ValueEnum)] -enum OutputFormat { - /// AWS credential_process JSON format - CredentialProcess, - /// Shell export statements - Env, -} - -#[tokio::main] -async fn main() { - let cli = Cli::parse(); - - match cli.command { - Commands::Login(args) => { - if let Err(e) = run_login(args).await { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - Commands::CredentialProcess(args) => { - if let Err(e) = run_credential_process(args) { - eprintln!("Error: {e}"); - std::process::exit(1); - } - } - } -} - -async fn run_login(args: LoginArgs) -> Result<(), String> { - // 1. OIDC Discovery - eprintln!("Discovering OIDC endpoints..."); - let endpoints = oidc::discover(&args.issuer).await?; - - // 2. Browser-based OIDC login - let id_token = oidc::login(&endpoints, &args.client_id, &args.scope, args.port).await?; - eprintln!("Authentication successful."); - - // 3. STS credential exchange - eprintln!("Exchanging token for credentials..."); - let creds = sts::assume_role(&args.proxy_url, &args.role_arn, &id_token, args.duration).await?; - - // 4. Cache credentials - let path = cache::write_credentials(&args.role_arn, &creds)?; - eprintln!("Credentials cached to {}", path.display()); - - // 5. Output - match args.format { - OutputFormat::CredentialProcess => output::print_credential_process(&creds), - OutputFormat::Env => output::print_env(&creds), - } - - Ok(()) -} - -fn run_credential_process(args: CredentialProcessArgs) -> Result<(), String> { - let creds = cache::read_credentials(&args.role_arn)? - .ok_or("No cached credentials found. Run 'source-coop login' first.")?; - - if cache::is_expired(&creds)? { - return Err( - "Cached credentials have expired. Run 'source-coop login' to refresh.".to_string(), - ); - } - - output::print_credential_process(&creds); - Ok(()) -} diff --git a/crates/cli/src/oidc.rs b/crates/cli/src/oidc.rs deleted file mode 100644 index adc94ef..0000000 --- a/crates/cli/src/oidc.rs +++ /dev/null @@ -1,230 +0,0 @@ -use base64::engine::general_purpose::URL_SAFE_NO_PAD; -use base64::Engine; -use rand::Rng; -use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use std::io::{BufRead, BufReader, Write}; -use tokio::net::TcpListener; -use url::Url; - -#[derive(Debug)] -pub struct OidcEndpoints { - pub authorization_endpoint: String, - pub token_endpoint: String, -} - -/// Fetch OIDC discovery document and extract endpoints. -pub async fn discover(issuer: &str) -> Result { - let discovery_url = format!( - "{}/.well-known/openid-configuration", - issuer.trim_end_matches('/') - ); - - let resp = reqwest::get(&discovery_url) - .await - .map_err(|e| format!("Failed to fetch OIDC discovery document: {e}"))?; - - if !resp.status().is_success() { - return Err(format!("OIDC discovery returned status {}", resp.status())); - } - - let doc: serde_json::Value = resp - .json() - .await - .map_err(|e| format!("Failed to parse OIDC discovery document: {e}"))?; - - let authorization_endpoint = doc["authorization_endpoint"] - .as_str() - .ok_or("Missing authorization_endpoint in discovery document")? - .to_string(); - - let token_endpoint = doc["token_endpoint"] - .as_str() - .ok_or("Missing token_endpoint in discovery document")? - .to_string(); - - Ok(OidcEndpoints { - authorization_endpoint, - token_endpoint, - }) -} - -/// Run the browser-based OAuth2 Authorization Code flow with PKCE. -/// Opens the user's browser to the OIDC provider, waits for the callback, -/// and returns the `id_token`. -pub async fn login( - endpoints: &OidcEndpoints, - client_id: &str, - scope: &str, - port: u16, -) -> Result { - let pkce = generate_pkce(); - let state: String = URL_SAFE_NO_PAD.encode(rand::thread_rng().gen::<[u8; 16]>()); - - // Bind local callback server - let listener = TcpListener::bind(format!("127.0.0.1:{port}")) - .await - .map_err(|e| format!("Failed to bind local server: {e}"))?; - - let local_addr = listener - .local_addr() - .map_err(|e| format!("Failed to get local address: {e}"))?; - let redirect_uri = format!("http://127.0.0.1:{}/callback", local_addr.port()); - - // Build authorization URL - let mut auth_url = Url::parse(&endpoints.authorization_endpoint) - .map_err(|e| format!("Invalid authorization endpoint URL: {e}"))?; - auth_url - .query_pairs_mut() - .append_pair("response_type", "code") - .append_pair("client_id", client_id) - .append_pair("redirect_uri", &redirect_uri) - .append_pair("scope", scope) - .append_pair("code_challenge", &pkce.challenge) - .append_pair("code_challenge_method", "S256") - .append_pair("state", &state); - - eprintln!("Opening browser for authentication..."); - if open::that(auth_url.as_str()).is_err() { - eprintln!( - "Could not open browser automatically. Please open this URL:\n{}", - auth_url - ); - } - - // Wait for callback - let (code, received_state) = wait_for_callback(&listener).await?; - - if received_state != state { - return Err("State mismatch — possible CSRF attack".to_string()); - } - - // Exchange code for tokens - exchange_code( - &endpoints.token_endpoint, - &code, - &redirect_uri, - client_id, - &pkce.verifier, - ) - .await -} - -/// Accept a single HTTP request on the callback listener, extract `code` and `state`. -async fn wait_for_callback(listener: &TcpListener) -> Result<(String, String), String> { - let (stream, _) = listener - .accept() - .await - .map_err(|e| format!("Failed to accept callback connection: {e}"))?; - - let std_stream = stream - .into_std() - .map_err(|e| format!("Failed to convert stream: {e}"))?; - std_stream - .set_nonblocking(false) - .map_err(|e| format!("Failed to set blocking: {e}"))?; - - let mut reader = BufReader::new(&std_stream); - let mut request_line = String::new(); - reader - .read_line(&mut request_line) - .map_err(|e| format!("Failed to read request: {e}"))?; - - let path = request_line - .split_whitespace() - .nth(1) - .ok_or("Invalid HTTP request")?; - - let url = Url::parse(&format!("http://localhost{path}")) - .map_err(|e| format!("Failed to parse callback URL: {e}"))?; - - let params: HashMap = url.query_pairs().into_owned().collect(); - - if let Some(error) = params.get("error") { - let desc = params - .get("error_description") - .map(|d| format!(": {d}")) - .unwrap_or_default(); - let html = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n\ -

Authentication Failed

{error}{desc}

\ -

You can close this tab.

" - ); - let _ = (&std_stream).write_all(html.as_bytes()); - return Err(format!("Authentication error: {error}{desc}")); - } - - let code = params - .get("code") - .ok_or("No authorization code in callback")? - .clone(); - let received_state = params.get("state").ok_or("No state in callback")?.clone(); - - let html = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n\ -

Authentication Successful

\ -

You can close this tab and return to your terminal.

"; - (&std_stream) - .write_all(html.as_bytes()) - .map_err(|e| format!("Failed to send response: {e}"))?; - - Ok((code, received_state)) -} - -/// Exchange authorization code for tokens, return the `id_token`. -async fn exchange_code( - token_endpoint: &str, - code: &str, - redirect_uri: &str, - client_id: &str, - code_verifier: &str, -) -> Result { - let client = reqwest::Client::new(); - let resp = client - .post(token_endpoint) - .form(&[ - ("grant_type", "authorization_code"), - ("code", code), - ("redirect_uri", redirect_uri), - ("client_id", client_id), - ("code_verifier", code_verifier), - ]) - .send() - .await - .map_err(|e| format!("Token exchange request failed: {e}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(format!("Token exchange failed (HTTP {status}): {body}")); - } - - let body: serde_json::Value = resp - .json() - .await - .map_err(|e| format!("Failed to parse token response: {e}"))?; - - body["id_token"] - .as_str() - .map(|s| s.to_string()) - .ok_or_else(|| "No id_token in token response".to_string()) -} - -struct Pkce { - verifier: String, - challenge: String, -} - -fn generate_pkce() -> Pkce { - let mut rng = rand::thread_rng(); - let bytes: Vec = (0..32).map(|_| rng.gen()).collect(); - let verifier = URL_SAFE_NO_PAD.encode(&bytes); - - let mut hasher = Sha256::new(); - hasher.update(verifier.as_bytes()); - let challenge = URL_SAFE_NO_PAD.encode(hasher.finalize()); - - Pkce { - verifier, - challenge, - } -} diff --git a/crates/cli/src/output.rs b/crates/cli/src/output.rs deleted file mode 100644 index 1307e8a..0000000 --- a/crates/cli/src/output.rs +++ /dev/null @@ -1,20 +0,0 @@ -use crate::sts::Credentials; - -/// Print credentials in AWS credential_process JSON format. -pub fn print_credential_process(creds: &Credentials) { - let json = serde_json::json!({ - "Version": 1, - "AccessKeyId": creds.access_key_id, - "SecretAccessKey": creds.secret_access_key, - "SessionToken": creds.session_token, - "Expiration": creds.expiration, - }); - println!("{}", serde_json::to_string_pretty(&json).unwrap()); -} - -/// Print credentials as shell export statements. -pub fn print_env(creds: &Credentials) { - println!("export AWS_ACCESS_KEY_ID={}", creds.access_key_id); - println!("export AWS_SECRET_ACCESS_KEY={}", creds.secret_access_key); - println!("export AWS_SESSION_TOKEN={}", creds.session_token); -} diff --git a/crates/cli/src/sts.rs b/crates/cli/src/sts.rs deleted file mode 100644 index 58a5306..0000000 --- a/crates/cli/src/sts.rs +++ /dev/null @@ -1,104 +0,0 @@ -use quick_xml::de::from_str as xml_from_str; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Credentials { - pub access_key_id: String, - pub secret_access_key: String, - pub session_token: String, - pub expiration: String, -} - -/// Call the proxy's STS AssumeRoleWithWebIdentity endpoint. -pub async fn assume_role( - proxy_url: &str, - role_arn: &str, - web_identity_token: &str, - duration_seconds: Option, -) -> Result { - let mut url = url::Url::parse(proxy_url).map_err(|e| format!("Invalid proxy URL: {e}"))?; - - url.query_pairs_mut() - .append_pair("Action", "AssumeRoleWithWebIdentity") - .append_pair("RoleArn", role_arn) - .append_pair("WebIdentityToken", web_identity_token); - - if let Some(duration) = duration_seconds { - url.query_pairs_mut() - .append_pair("DurationSeconds", &duration.to_string()); - } - - let resp = reqwest::get(url.as_str()) - .await - .map_err(|e| format!("STS request failed: {e}"))?; - - let status = resp.status(); - let body = resp - .text() - .await - .map_err(|e| format!("Failed to read STS response: {e}"))?; - - if !status.is_success() { - // Try to parse error XML for a better message - if let Ok(err) = xml_from_str::(&body) { - return Err(format!( - "STS error ({}): {}", - err.error.code, err.error.message - )); - } - return Err(format!("STS request failed (HTTP {status}): {body}")); - } - - let parsed: StsResponse = - xml_from_str(&body).map_err(|e| format!("Failed to parse STS response XML: {e}"))?; - - let creds = parsed.result.credentials; - Ok(Credentials { - access_key_id: creds.access_key_id, - secret_access_key: creds.secret_access_key, - session_token: creds.session_token, - expiration: creds.expiration, - }) -} - -// XML deserialization types matching the STS response format - -#[derive(Debug, Deserialize)] -#[serde(rename = "AssumeRoleWithWebIdentityResponse")] -struct StsResponse { - #[serde(rename = "AssumeRoleWithWebIdentityResult")] - result: StsResult, -} - -#[derive(Debug, Deserialize)] -struct StsResult { - #[serde(rename = "Credentials")] - credentials: StsCredentials, -} - -#[derive(Debug, Deserialize)] -struct StsCredentials { - #[serde(rename = "AccessKeyId")] - access_key_id: String, - #[serde(rename = "SecretAccessKey")] - secret_access_key: String, - #[serde(rename = "SessionToken")] - session_token: String, - #[serde(rename = "Expiration")] - expiration: String, -} - -#[derive(Debug, Deserialize)] -#[serde(rename = "ErrorResponse")] -struct StsErrorResponse { - #[serde(rename = "Error")] - error: StsError, -} - -#[derive(Debug, Deserialize)] -struct StsError { - #[serde(rename = "Code")] - code: String, - #[serde(rename = "Message")] - message: String, -} From beaa3a9076551c6e4547d39b5a0b0bd51bd2be29 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 1 Mar 2026 17:40:04 -0500 Subject: [PATCH 64/82] feat: add VitePress documentation site (#110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Adds a comprehensive VitePress documentation site in `docs/` covering authentication, configuration, deployment, architecture, and extension points - Organized into user-facing guide (accessing data) and admin-facing sections (deploying/configuring the proxy) - Styled to match docs.source.coop visual identity: IBM Plex Sans body, Cascadia Mono headings, warm off-white/teal-gray color scheme ## Test plan - [ ] `cd docs && pnpm install && pnpm docs:dev` — site builds and serves locally - [ ] Navigate all sidebar links — no broken links - [ ] Mermaid diagrams render correctly - [ ] Light and dark themes both render properly - [ ] Code examples are syntactically valid 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 --- .claude/launch.json | 12 + .github/workflows/docs.yaml | 59 + .gitignore | 3 + docs/.vitepress/config.ts | 166 ++ docs/.vitepress/theme/index.ts | 4 + docs/.vitepress/theme/style.css | 220 ++ docs/architecture/crate-layout.md | 114 + docs/architecture/index.md | 57 + docs/architecture/multi-runtime.md | 78 + docs/architecture/request-lifecycle.md | 95 + docs/auth/backend-auth.md | 255 ++ docs/auth/index.md | 46 + docs/auth/proxy-auth.md | 353 +++ docs/auth/sealed-tokens.md | 67 + docs/configuration/buckets.md | 136 + docs/configuration/credentials.md | 66 + docs/configuration/index.md | 68 + docs/configuration/providers/cached.md | 42 + docs/configuration/providers/dynamodb.md | 33 + docs/configuration/providers/http.md | 39 + docs/configuration/providers/index.md | 57 + docs/configuration/providers/postgres.md | 28 + docs/configuration/providers/static-file.md | 86 + docs/configuration/roles.md | 149 ++ docs/deployment/cloudflare-workers.md | 95 + docs/deployment/index.md | 13 + docs/deployment/server.md | 81 + docs/extending/custom-backend.md | 120 + docs/extending/custom-provider.md | 104 + docs/extending/custom-resolver.md | 128 + docs/extending/index.md | 17 + docs/getting-started/index.md | 63 + docs/getting-started/local-development.md | 105 + docs/guide/authentication.md | 135 + docs/guide/client-usage.md | 109 + docs/guide/endpoints.md | 102 + docs/guide/index.md | 24 + docs/index.md | 79 + docs/package.json | 20 + docs/pnpm-lock.yaml | 2620 +++++++++++++++++++ docs/reference/config-example.md | 164 ++ docs/reference/errors.md | 58 + docs/reference/index.md | 5 + docs/reference/operations.md | 43 + 44 files changed, 6318 insertions(+) create mode 100644 .claude/launch.json create mode 100644 .github/workflows/docs.yaml create mode 100644 docs/.vitepress/config.ts create mode 100644 docs/.vitepress/theme/index.ts create mode 100644 docs/.vitepress/theme/style.css create mode 100644 docs/architecture/crate-layout.md create mode 100644 docs/architecture/index.md create mode 100644 docs/architecture/multi-runtime.md create mode 100644 docs/architecture/request-lifecycle.md create mode 100644 docs/auth/backend-auth.md create mode 100644 docs/auth/index.md create mode 100644 docs/auth/proxy-auth.md create mode 100644 docs/auth/sealed-tokens.md create mode 100644 docs/configuration/buckets.md create mode 100644 docs/configuration/credentials.md create mode 100644 docs/configuration/index.md create mode 100644 docs/configuration/providers/cached.md create mode 100644 docs/configuration/providers/dynamodb.md create mode 100644 docs/configuration/providers/http.md create mode 100644 docs/configuration/providers/index.md create mode 100644 docs/configuration/providers/postgres.md create mode 100644 docs/configuration/providers/static-file.md create mode 100644 docs/configuration/roles.md create mode 100644 docs/deployment/cloudflare-workers.md create mode 100644 docs/deployment/index.md create mode 100644 docs/deployment/server.md create mode 100644 docs/extending/custom-backend.md create mode 100644 docs/extending/custom-provider.md create mode 100644 docs/extending/custom-resolver.md create mode 100644 docs/extending/index.md create mode 100644 docs/getting-started/index.md create mode 100644 docs/getting-started/local-development.md create mode 100644 docs/guide/authentication.md create mode 100644 docs/guide/client-usage.md create mode 100644 docs/guide/endpoints.md create mode 100644 docs/guide/index.md create mode 100644 docs/index.md create mode 100644 docs/package.json create mode 100644 docs/pnpm-lock.yaml create mode 100644 docs/reference/config-example.md create mode 100644 docs/reference/errors.md create mode 100644 docs/reference/index.md create mode 100644 docs/reference/operations.md diff --git a/.claude/launch.json b/.claude/launch.json new file mode 100644 index 0000000..2d3c45e --- /dev/null +++ b/.claude/launch.json @@ -0,0 +1,12 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "docs", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["docs:dev"], + "port": 5173, + "cwd": "docs" + } + ] +} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..20c2462 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,59 @@ +name: Deploy Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - ".github/workflows/docs.yaml" + + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: docs/pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: docs + + - name: Build docs + run: pnpm docs:build + working-directory: docs + + - uses: actions/upload-pages-artifact@v3 + with: + path: docs/.vitepress/dist + + deploy: + name: Deploy + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/deploy-pages@v4 + id: deployment diff --git a/.gitignore b/.gitignore index a55e53a..abaabb6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ scripts/task_definition.json target .wrangler .env* +node_modules +docs/.vitepress/cache +docs/.vitepress/dist diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts new file mode 100644 index 0000000..bd1c24a --- /dev/null +++ b/docs/.vitepress/config.ts @@ -0,0 +1,166 @@ +import { defineConfig } from "vitepress"; +import { withMermaid } from "vitepress-plugin-mermaid"; + +const adminSidebar = [ + { + text: "Getting Started", + items: [ + { text: "Quick Start", link: "/getting-started/" }, + { + text: "Local Development", + link: "/getting-started/local-development", + }, + ], + }, + { + text: "Configuration", + items: [ + { text: "Overview", link: "/configuration/" }, + { text: "Buckets", link: "/configuration/buckets" }, + { text: "Roles", link: "/configuration/roles" }, + { text: "Credentials", link: "/configuration/credentials" }, + { + text: "Providers", + collapsed: false, + items: [ + { text: "Overview", link: "/configuration/providers/" }, + { + text: "Static File", + link: "/configuration/providers/static-file", + }, + { text: "HTTP API", link: "/configuration/providers/http" }, + { + text: "DynamoDB", + link: "/configuration/providers/dynamodb", + }, + { + text: "PostgreSQL", + link: "/configuration/providers/postgres", + }, + { + text: "Caching", + link: "/configuration/providers/cached", + }, + ], + }, + ], + }, + { + text: "Authentication", + items: [ + { text: "Overview", link: "/auth/" }, + { + text: "Client Auth (OIDC/STS)", + link: "/auth/proxy-auth", + }, + { + text: "Backend Auth", + link: "/auth/backend-auth", + }, + { text: "Sealed Session Tokens", link: "/auth/sealed-tokens" }, + ], + }, + { + text: "Deployment", + items: [ + { text: "Overview", link: "/deployment/" }, + { text: "Server Runtime", link: "/deployment/server" }, + { + text: "Cloudflare Workers", + link: "/deployment/cloudflare-workers", + }, + ], + }, + { + text: "Architecture", + items: [ + { text: "Overview", link: "/architecture/" }, + { text: "Crate Layout", link: "/architecture/crate-layout" }, + { + text: "Request Lifecycle", + link: "/architecture/request-lifecycle", + }, + { + text: "Multi-Runtime Design", + link: "/architecture/multi-runtime", + }, + ], + }, + { + text: "Extending", + items: [ + { text: "Overview", link: "/extending/" }, + { text: "Custom Resolver", link: "/extending/custom-resolver" }, + { text: "Custom Provider", link: "/extending/custom-provider" }, + { text: "Custom Backend", link: "/extending/custom-backend" }, + ], + }, +]; + +export default withMermaid( + defineConfig({ + title: "Source Data Proxy", + description: "Multi-runtime S3 gateway proxy in Rust", + + themeConfig: { + nav: [ + { text: "User Guide", link: "/guide/" }, + { text: "Administration", link: "/getting-started/" }, + { text: "Reference", link: "/reference/" }, + ], + + sidebar: { + "/guide/": [ + { + text: "User Guide", + items: [ + { text: "Overview", link: "/guide/" }, + { text: "Endpoints", link: "/guide/endpoints" }, + { text: "Authentication", link: "/guide/authentication" }, + { text: "Client Usage", link: "/guide/client-usage" }, + ], + }, + ], + + "/getting-started/": adminSidebar, + "/configuration/": adminSidebar, + "/auth/": adminSidebar, + "/deployment/": adminSidebar, + "/architecture/": adminSidebar, + "/extending/": adminSidebar, + + "/reference/": [ + { + text: "Reference", + items: [ + { text: "Overview", link: "/reference/" }, + { + text: "Supported Operations", + link: "/reference/operations", + }, + { text: "Error Codes", link: "/reference/errors" }, + { text: "Config Example", link: "/reference/config-example" }, + ], + }, + ], + }, + + socialLinks: [ + { + icon: "github", + link: "https://github.com/source-cooperative/data.source.coop", + }, + ], + + search: { + provider: "local", + }, + + footer: { + message: "Released under the MIT / Apache-2.0 License.", + copyright: + 'A Radiant Earth project. Copyright © 2026 Source Cooperative.', + }, + }, + }), +); diff --git a/docs/.vitepress/theme/index.ts b/docs/.vitepress/theme/index.ts new file mode 100644 index 0000000..617deeb --- /dev/null +++ b/docs/.vitepress/theme/index.ts @@ -0,0 +1,4 @@ +import DefaultTheme from "vitepress/theme"; +import "./style.css"; + +export default DefaultTheme; diff --git a/docs/.vitepress/theme/style.css b/docs/.vitepress/theme/style.css new file mode 100644 index 0000000..4bd08e0 --- /dev/null +++ b/docs/.vitepress/theme/style.css @@ -0,0 +1,220 @@ +/** + * Source Cooperative theme + * + * Matches docs.source.coop visual identity: + * - Font: IBM Plex Sans + * - Mono: Berkeley Mono (with fallbacks) + * - Light: warm off-white #efebea background, dark teal-gray #2c3233 text + * - Dark: inverted — #2c3233 background, #efebea text + * + * Reference: github.com/source-cooperative/docs.source.coop/blob/main/src/css/custom.css + */ + +/* ------------------------------------------------------------------ */ +/* Typography */ +/* ------------------------------------------------------------------ */ + +@import url("https://fonts.googleapis.com/css2?family=Cascadia+Mono:wght@400;600&family=IBM+Plex+Sans:ital,wght@0,100..700;1,100..700&family=IBM+Plex+Mono:ital,wght@0,400;0,500;0,600;1,400&display=swap"); + +:root { + --vp-font-family-base: "IBM Plex Sans", system-ui, -apple-system, sans-serif; + --vp-font-family-mono: "IBM Plex Mono", SFMono-Regular, "SF Mono", Menlo, + Consolas, "Liberation Mono", monospace; + --vp-font-family-heading: "Cascadia Mono", var(--vp-font-family-mono); +} + +/* ------------------------------------------------------------------ */ +/* Light theme colors */ +/* Warm off-white background, dark teal-gray primary */ +/* ------------------------------------------------------------------ */ + +:root { + /* Primary — dark teal-gray (same as docs.source.coop) */ + --vp-c-brand-1: #2c3233; + --vp-c-brand-2: #303738; + --vp-c-brand-3: #394142; + --vp-c-brand-soft: rgba(44, 50, 51, 0.08); + + /* Page background — warm off-white */ + --vp-c-bg: #efebea; + --vp-c-bg-alt: #e6e1df; + --vp-c-bg-elv: #ffffff; + --vp-c-bg-soft: #e6e1df; + + /* Text */ + --vp-c-text-1: #2c3233; + --vp-c-text-2: rgba(44, 50, 51, 0.78); + --vp-c-text-3: rgba(44, 50, 51, 0.56); + + /* Borders & dividers */ + --vp-c-divider: rgba(44, 50, 51, 0.12); + --vp-c-gutter: rgba(44, 50, 51, 0.06); + --vp-c-border: rgba(44, 50, 51, 0.15); + + /* Tip callout — inherit brand */ + --vp-c-tip-1: var(--vp-c-brand-1); + --vp-c-tip-2: var(--vp-c-brand-2); + --vp-c-tip-3: var(--vp-c-brand-3); + --vp-c-tip-soft: var(--vp-c-brand-soft); + + /* Navbar / sidebar */ + --vp-nav-bg-color: #ffffff; + --vp-sidebar-bg-color: #f7f4f3; + + /* Code blocks */ + --vp-code-block-bg: #f6f7f8; + --vp-code-bg: rgba(44, 50, 51, 0.06); +} + +/* ------------------------------------------------------------------ */ +/* Dark theme colors */ +/* Inverted: teal-gray background, warm off-white primary */ +/* ------------------------------------------------------------------ */ + +.dark { + --vp-c-brand-1: #efebea; + --vp-c-brand-2: #dbd1cf; + --vp-c-brand-3: #b29e99; + --vp-c-brand-soft: rgba(239, 235, 234, 0.1); + + /* Page background */ + --vp-c-bg: #2c3233; + --vp-c-bg-alt: #242a2b; + --vp-c-bg-elv: #343b3c; + --vp-c-bg-soft: #343b3c; + + /* Text */ + --vp-c-text-1: #efebea; + --vp-c-text-2: rgba(239, 235, 234, 0.72); + --vp-c-text-3: rgba(239, 235, 234, 0.48); + + /* Borders & dividers */ + --vp-c-divider: rgba(239, 235, 234, 0.12); + --vp-c-gutter: rgba(239, 235, 234, 0.06); + --vp-c-border: rgba(239, 235, 234, 0.15); + + /* Navbar / sidebar */ + --vp-nav-bg-color: #242a2b; + --vp-sidebar-bg-color: #272e2f; + + /* Code blocks */ + --vp-code-block-bg: rgba(255, 255, 255, 0.06); + --vp-code-bg: rgba(255, 255, 255, 0.1); +} + +/* ------------------------------------------------------------------ */ +/* Buttons */ +/* ------------------------------------------------------------------ */ + +:root { + --vp-button-brand-border: transparent; + --vp-button-brand-text: #efebea; + --vp-button-brand-bg: #2c3233; + --vp-button-brand-hover-border: transparent; + --vp-button-brand-hover-text: #efebea; + --vp-button-brand-hover-bg: #394142; + --vp-button-brand-active-border: transparent; + --vp-button-brand-active-text: #efebea; + --vp-button-brand-active-bg: #1f2324; +} + +.dark { + --vp-button-brand-text: #2c3233; + --vp-button-brand-bg: #efebea; + --vp-button-brand-hover-text: #2c3233; + --vp-button-brand-hover-bg: #ffffff; + --vp-button-brand-active-text: #2c3233; + --vp-button-brand-active-bg: #dbd1cf; +} + +/* ------------------------------------------------------------------ */ +/* Home hero */ +/* ------------------------------------------------------------------ */ + +:root { + --vp-home-hero-name-color: #2c3233; + --vp-home-hero-name-background: none; + --vp-home-hero-image-background-image: none; + --vp-home-hero-image-filter: none; +} + +.dark { + --vp-home-hero-name-color: #efebea; +} + +/* ------------------------------------------------------------------ */ +/* Headings — Cascadia Mono for a distinctive technical feel */ +/* ------------------------------------------------------------------ */ + +.vp-doc h1, +.vp-doc h2, +.vp-doc h3, +.vp-doc h4, +.vp-doc h5, +.vp-doc h6 { + font-family: var(--vp-font-family-heading); + letter-spacing: -0.02em; +} + +/* Hero heading on homepage */ +.VPHero .name, +.VPHero .text { + font-family: var(--vp-font-family-heading) !important; + letter-spacing: -0.03em; +} + +/* Feature card titles */ +.VPFeature .title { + font-family: var(--vp-font-family-heading); +} + +/* Sidebar group headings */ +.VPSidebarItem.level-0 > .item > .text { + font-family: var(--vp-font-family-heading); +} + +/* ------------------------------------------------------------------ */ +/* Content links — underlined, matching docs.source.coop */ +/* ------------------------------------------------------------------ */ + +.vp-doc a { + text-decoration: underline; + text-underline-offset: 2px; +} + +.vp-doc a:hover { + text-decoration: none; +} + +/* ------------------------------------------------------------------ */ +/* Navbar — clean white surface (light) / dark surface (dark) */ +/* ------------------------------------------------------------------ */ + +.VPNav { + background-color: var(--vp-nav-bg-color) !important; +} + +.VPNavBar { + background-color: var(--vp-nav-bg-color) !important; + border-bottom: 1px solid var(--vp-c-divider) !important; +} + +.VPNavBar .divider { + display: none; +} + +/* ------------------------------------------------------------------ */ +/* Sidebar — subtle background */ +/* ------------------------------------------------------------------ */ + +.VPSidebar { + background-color: var(--vp-sidebar-bg-color) !important; +} + +/* ------------------------------------------------------------------ */ +/* Code font size — match docs.source.coop 95% */ +/* ------------------------------------------------------------------ */ + +:root { + --vp-code-font-size: 95%; +} diff --git a/docs/architecture/crate-layout.md b/docs/architecture/crate-layout.md new file mode 100644 index 0000000..3870f84 --- /dev/null +++ b/docs/architecture/crate-layout.md @@ -0,0 +1,114 @@ +# Crate Layout + +The project is organized as a Cargo workspace with libraries (traits and logic) and runtimes (executable targets). + +``` +crates/ +├── cli/ # source-coop CLI (OIDC login → STS credential exchange) +├── libs/ # Libraries — not directly runnable +│ ├── core/ (source-coop-core) # Runtime-agnostic: traits, S3 parsing, SigV4, config +│ ├── sts/ (source-coop-sts) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) +│ ├── oidc-provider/ # Outbound OIDC provider (JWT signing, JWKS, exchange) +│ └── source-coop/ # Source Cooperative resolver and API client +└── runtimes/ # Runnable targets — one per deployment platform + ├── server/ (source-coop-server) # Tokio/Hyper for container deployments + └── cf-workers/ # Cloudflare Workers for edge deployments +``` + +## Crate Responsibilities + +### `source-coop-core` + +The runtime-agnostic core. Contains: +- `ProxyHandler` — Two-phase request handler (`resolve_request()` → `HandlerAction`) +- `RequestResolver` and `DefaultResolver` — Request parsing, SigV4 auth, authorization +- `ConfigProvider` trait and implementations (static file, HTTP, DynamoDB, Postgres) +- `ProxyBackend` trait — Runtime abstraction for store/signer/raw HTTP +- S3 request parsing, XML response building, list prefix rewriting +- SigV4 signature verification +- Sealed session token encryption/decryption +- Type definitions (`BucketConfig`, `RoleConfig`, `AccessScope`, etc.) + +**Feature flags:** +- `config-http` — HTTP API config provider +- `config-dynamodb` — DynamoDB config provider +- `config-postgres` — PostgreSQL config provider +- `azure` — Azure Blob Storage support +- `gcp` — Google Cloud Storage support + +### `source-coop-sts` + +OIDC token exchange implementing `AssumeRoleWithWebIdentity`: +- JWT decoding and validation (RS256) +- JWKS fetching and caching +- Trust policy evaluation (issuer, audience, subject conditions) +- Temporary credential minting with scope template variables + +### `source-coop-oidc-provider` + +Outbound OIDC identity provider for backend authentication: +- RSA JWT signing (`JwtSigner`) +- JWKS endpoint serving +- OpenID Connect discovery document +- AWS credential exchange (`AwsOidcBackendAuth`) +- Credential caching + +### `source-coop-server` + +The native server runtime: +- Tokio/Hyper HTTP server +- `ServerBackend` implementing `ProxyBackend` with reqwest +- Streaming via hyper `Incoming` bodies and reqwest `bytes_stream()` +- CLI argument parsing (`--config`, `--listen`, `--domain`, `--sts-config`) + +### `source-coop-cf-workers` + +The Cloudflare Workers WASM runtime: +- `WorkerBackend` implementing `ProxyBackend` with `web_sys::fetch` +- `FetchConnector` bridging `object_store` HTTP to Workers Fetch API +- JS `ReadableStream` passthrough for zero-copy streaming +- Config loading from env vars (`PROXY_CONFIG`) + +::: warning +This crate is excluded from the workspace `default-members` because WASM types are `!Send` and won't compile on native targets. Always build with `--target wasm32-unknown-unknown`. +::: + +### `source-coop` (lib) + +Source Cooperative-specific resolver and API client: +- `SourceCoopResolver` — Custom namespace mapping (`/{account}/{repo}/{key}`) +- External auth via Source Cooperative API + +### `cli` + +Command-line tool for OIDC authentication: +- Browser-based OAuth2 Authorization Code + PKCE flow +- `credential_process` integration with AWS SDKs +- Credential caching in `~/.source-coop/credentials/` + +## Dependency Flow + +```mermaid +flowchart TD + core["source-coop-core"] + sts["source-coop-sts"] + oidc["source-coop-oidc-provider"] + api["source-coop (lib)"] + server["source-coop-server"] + workers["source-coop-cf-workers"] + cli["source-coop CLI"] + + server --> core + server --> sts + server --> oidc + workers --> core + workers --> sts + workers --> oidc + workers --> api + cli --> sts + sts --> core + oidc --> core + api --> core +``` + +Libraries define trait abstractions. Runtimes implement `ProxyBackend` with platform-native primitives and wire everything together. diff --git a/docs/architecture/index.md b/docs/architecture/index.md new file mode 100644 index 0000000..652ccb0 --- /dev/null +++ b/docs/architecture/index.md @@ -0,0 +1,57 @@ +# Architecture Overview + +The Source Data Proxy is an S3-compliant gateway that sits between clients and backend object stores. It provides authentication, authorization, and transparent proxying with zero-copy streaming. + +## High-Level Architecture + +```mermaid +flowchart LR + Clients["S3 Clients
(aws-cli, boto3, SDKs)"] + + subgraph Proxy["source-coop-proxy"] + Resolver["Request Resolver
(parse, auth, authorize)"] + Handler["Proxy Handler
(dispatch operations)"] + Backend["Proxy Backend
(runtime-specific I/O)"] + end + + Config["Config Provider
(Static, HTTP, DynamoDB, Postgres)"] + OIDC["OIDC Providers
(Auth0, GitHub, Keycloak)"] + Stores["Object Stores
(S3, MinIO, R2, Azure, GCS)"] + + Clients <--> Resolver + Resolver <--> Config + Resolver <--> OIDC + Handler <--> Backend + Backend <--> Stores +``` + +## Design Principles + +**Runtime-agnostic core** — The core proxy logic (`source-coop-core`) has zero runtime dependencies. No Tokio, no `worker-rs`. It compiles to both native and WASM targets. + +**Two-phase handler** — The proxy handler separates request resolution from execution. `resolve_request()` determines what to do; the runtime executes it. This keeps streaming logic in runtime-specific code where it belongs. + +**Presigned URLs for streaming** — GET, HEAD, PUT, and DELETE operations use presigned URLs. The runtime forwards the request directly to the backend — no buffering, no double-handling of bodies. + +**Pluggable traits** — Three trait boundaries enable customization: +- `RequestResolver` — How requests are parsed, authenticated, and authorized +- `ConfigProvider` — Where configuration comes from +- `ProxyBackend` — How the runtime interacts with backends + +## Key Components + +| Component | Crate | Responsibility | +|-----------|-------|---------------| +| [Proxy Handler](./request-lifecycle) | `core` | Dispatch operations via presigned URLs, LIST, or multipart | +| [Request Resolver](./request-lifecycle#request-resolution) | `core` | Parse S3 requests, authenticate, authorize | +| [Config Providers](/configuration/providers/) | `core` | Load buckets, roles, credentials | +| [STS Handler](/auth/proxy-auth#oidcsts-temporary-credentials) | `sts` | OIDC token exchange, credential minting | +| [OIDC Provider](/auth/backend-auth#oidc-backend-auth) | `oidc-provider` | Self-signed JWT minting, backend credential exchange | +| [Server Runtime](./multi-runtime#server-runtime) | `server` | Tokio/Hyper HTTP server | +| [Workers Runtime](./multi-runtime#cloudflare-workers-runtime) | `cf-workers` | WASM-based Cloudflare Workers | + +## Further Reading + +- [Crate Layout](./crate-layout) — How the workspace is organized +- [Request Lifecycle](./request-lifecycle) — How a request flows through the proxy +- [Multi-Runtime Design](./multi-runtime) — How the same core runs on native and WASM diff --git a/docs/architecture/multi-runtime.md b/docs/architecture/multi-runtime.md new file mode 100644 index 0000000..94e6912 --- /dev/null +++ b/docs/architecture/multi-runtime.md @@ -0,0 +1,78 @@ +# Multi-Runtime Design + +The proxy runs on two runtimes — a native Tokio/Hyper server for container deployments and Cloudflare Workers for edge deployments. The same core logic compiles to both targets through careful abstraction of platform-specific concerns. + +## Runtime Comparison + +| | Server Runtime | CF Workers Runtime | +|---|---|---| +| **Platform** | Linux/macOS containers | Cloudflare Workers (V8) | +| **Target** | `x86_64` / `aarch64` | `wasm32-unknown-unknown` | +| **HTTP client** | reqwest | `web_sys::fetch` | +| **Streaming** | hyper `Incoming` / reqwest `bytes_stream()` | JS `ReadableStream` passthrough | +| **Object store connector** | Default (reqwest-based) | `FetchConnector` | +| **Backend support** | S3, Azure, GCS | S3 only | +| **Config loading** | TOML file | Env var (JSON or JS object) | +| **Threading** | Multi-threaded (`Send + Sync` required) | Single-threaded (`!Send` types allowed) | + +## How It Works + +### MaybeSend / MaybeSync + +The core challenge is that Tokio requires `Send + Sync` for task spawning, while WASM runtimes are single-threaded and use `!Send` types (like `JsValue` and `ReadableStream`). + +The solution is conditional trait aliases defined in `source-coop-core`: + +- On native targets: `MaybeSend` resolves to `Send`, `MaybeSync` resolves to `Sync` +- On `wasm32`: `MaybeSend` and `MaybeSync` are blanket traits that every type implements + +All core traits (`ProxyBackend`, `RequestResolver`, `ConfigProvider`) use `MaybeSend + MaybeSync` instead of `Send + Sync`, so they compile on both targets. + +The `Signer` trait from `object_store` requires real `Send + Sync`, which works because `UnsignedUrlSigner` only holds `String` fields, and `object_store`'s built-in store types are `Send + Sync`. + +### RPITIT Async Methods + +Core traits use return-position `impl Trait` in trait (RPITIT) for async methods instead of `#[async_trait]`: + +```rust +pub trait RequestResolver: Clone + MaybeSend + MaybeSync + 'static { + fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> impl Future> + MaybeSend; +} +``` + +This avoids `#[async_trait]`'s `Box` requirement, which won't compile on WASM targets. + +## Server Runtime + +The server runtime (`crates/runtimes/server/`) uses Tokio and Hyper: + +- **Forward actions**: reqwest sends the presigned URL request. For GET, the response body is streamed via `bytes_stream()`. For PUT, the client's hyper `Incoming` body is streamed directly to reqwest. +- **`ServerBackend`**: Creates `object_store` instances with the default HTTP connector (reqwest) and uses reqwest for `send_raw()` (multipart). + +## Cloudflare Workers Runtime + +The CF Workers runtime (`crates/runtimes/cf-workers/`) uses `worker-rs`, `wasm-bindgen`, and `web_sys`: + +- **Forward actions**: JS `ReadableStream` bodies pass through without touching Rust. The Workers Fetch API handles streaming natively. +- **`WorkerBackend`**: Creates `object_store` instances with `FetchConnector` injected for HTTP transport. + +### FetchConnector + +`FetchConnector` bridges `object_store`'s `HttpConnector` trait to the Workers Fetch API. Since `worker::Fetch::send()` is `!Send`, each call is wrapped in `spawn_local` with a oneshot channel to bridge back to the `Send` context that `object_store` expects. + +This is only used for LIST operations — presigned URL operations bypass `object_store` entirely. + +### WASM Limitations + +- **S3 only**: Azure and GCS builders are gated behind cargo features that are disabled for the Workers runtime +- **`Instant::now()` panics on WASM**: The `UnsignedUrlSigner` avoids the `InstanceCredentialProvider` → `TokenCache` → `Instant::now()` code path that panics on WASM +- **No `default-members`**: The CF Workers crate is excluded from the workspace default members. Always build with: + ```bash + cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown + ``` diff --git a/docs/architecture/request-lifecycle.md b/docs/architecture/request-lifecycle.md new file mode 100644 index 0000000..05556f8 --- /dev/null +++ b/docs/architecture/request-lifecycle.md @@ -0,0 +1,95 @@ +# Request Lifecycle + +Every S3 request flows through a two-phase dispatch model: first the request is resolved (parsed, authenticated, authorized), then the appropriate action is executed by the runtime. + +## Overview + +```mermaid +sequenceDiagram + participant Client + participant Runtime as Runtime
(Server or Workers) + participant Resolver as Request Resolver + participant Handler as Proxy Handler + participant Backend as Backend Store + + Client->>Runtime: HTTP request + Runtime->>Handler: resolve_request(method, path, query, headers) + Handler->>Resolver: resolve(method, path, query, headers) + Resolver->>Resolver: Parse S3 operation + Resolver->>Resolver: Authenticate (SigV4) + Resolver->>Resolver: Authorize (check scopes) + Resolver-->>Handler: ResolvedAction::Proxy + Handler->>Handler: Dispatch operation + Handler-->>Runtime: HandlerAction + + alt Forward (GET/HEAD/PUT/DELETE) + Runtime->>Backend: Execute presigned URL + Backend-->>Runtime: Stream response + Runtime-->>Client: Stream response + else Response (LIST, errors) + Runtime-->>Client: Return response body + else NeedsBody (multipart) + Runtime->>Runtime: Collect request body + Runtime->>Handler: handle_with_body(pending, body) + Handler->>Backend: Signed multipart request + Backend-->>Handler: Response + Handler-->>Runtime: ProxyResult + Runtime-->>Client: Response + end +``` + +## Phase 1: Request Resolution + +The `RequestResolver` determines what to do with an incoming request. The `DefaultResolver` handles standard S3 proxy behavior: + +1. **Parse the S3 operation** from the HTTP method, path, query, and headers + - Path-style: `GET /bucket/key` → GetObject on `bucket` with key `key` + - Virtual-hosted: `GET /key` with `Host: bucket.s3.example.com` → same operation +2. **Authenticate** the request by verifying the SigV4 signature against stored or sealed credentials +3. **Authorize** by checking the caller's access scopes against the requested bucket, key prefix, and operation +4. **Return** a `ResolvedAction`: + - `Proxy { operation, bucket_config, list_rewrite }` — forward to a backend + - `Response { status, headers, body }` — return a synthetic response (e.g., `ListBuckets`) + +Custom resolvers can implement entirely different routing, authentication, and namespace mapping. + +## Phase 2: Handler Dispatch + +The `ProxyHandler` takes the resolved action and dispatches it based on the S3 operation type. It returns a `HandlerAction` enum: + +### `Forward(ForwardRequest)` + +Used for: **GET, HEAD, PUT, DELETE** + +The handler generates a presigned URL using the backend's `Signer` and returns it to the runtime with filtered headers. The runtime executes the presigned URL with its native HTTP client, streaming request and response bodies directly. The handler never touches the body data. + +- Presigned URL TTL: 300 seconds +- Headers forwarded: `range`, `if-match`, `if-none-match`, `if-modified-since`, `if-unmodified-since`, `content-type`, `content-length`, `content-md5`, `content-encoding`, `content-disposition`, `cache-control`, `x-amz-content-sha256` + +### `Response(ProxyResult)` + +Used for: **LIST, errors, synthetic responses** + +For LIST operations, the handler calls `object_store::list_with_delimiter()` via the backend's store, builds S3 `ListObjectsV2` XML from the results, and returns it as a complete response. If a `ListRewrite` is configured, key prefixes are transformed in the XML. + +::: info +LIST returns all results in a single response. `IsTruncated` is always `false`. The proxy does not support S3-style pagination with continuation tokens. +::: + +### `NeedsBody(PendingRequest)` + +Used for: **CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload** + +Multipart operations need the request body (e.g., the XML body for `CompleteMultipartUpload`). The runtime materializes the body, then calls `handler.handle_with_body()`, which signs the request using `S3RequestSigner` and sends it via `backend.send_raw()`. + +::: warning +Multipart uploads are only supported for `backend_type = "s3"`. Non-S3 backends should use single PUT requests (object_store handles chunking internally). +::: + +## Response Header Forwarding + +The proxy forwards only specific headers from the backend response to the client: + +`content-type`, `content-length`, `content-range`, `etag`, `last-modified`, `accept-ranges`, `content-encoding`, `content-disposition`, `cache-control`, `x-amz-request-id`, `x-amz-version-id`, `location` + +All other backend headers are filtered out. diff --git a/docs/auth/backend-auth.md b/docs/auth/backend-auth.md new file mode 100644 index 0000000..d6e68ce --- /dev/null +++ b/docs/auth/backend-auth.md @@ -0,0 +1,255 @@ +# Authenticating with Object Store Backends + +The proxy needs credentials to access backend object stores (S3, Azure Blob Storage, GCS). There are two approaches: static credentials stored in the proxy config, and OIDC-based credential resolution where the proxy acts as its own identity provider. + +## Static Backend Credentials + +The simplest approach is to include credentials directly in the bucket's `backend_options`: + +```toml +[[buckets]] +name = "my-data" +backend_type = "s3" + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" +access_key_id = "AKIAIOSFODNN7EXAMPLE" +secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +``` + +This works for any backend type. For anonymous backend access (e.g., public buckets), omit the credential fields and add `skip_signature = "true"`. + +## OIDC Backend Auth + +For production deployments, the proxy can act as its own OIDC identity provider. Instead of storing long-lived backend credentials, the proxy mints self-signed JWTs and exchanges them with cloud providers for temporary credentials — the same pattern used by GitHub Actions and Vercel for AWS access. + +### How It Works + +```mermaid +sequenceDiagram + participant Client as S3 Client + participant Proxy as Data Proxy + participant Cloud as Cloud Provider STS
(e.g., AWS STS) + participant Store as Object Store
(e.g., S3) + + Client->>Proxy: 1. S3 request (e.g., GET /bucket/key) + Proxy->>Proxy: 2. Authenticate client + Proxy->>Proxy: 3. Bucket has auth_type=oidc
Mint self-signed JWT + Proxy->>Cloud: 4. AssumeRoleWithWebIdentity
(JWT + Role ARN) + Cloud->>Proxy: 5. Fetch /.well-known/jwks.json + Proxy-->>Cloud: 6. RSA public key + Cloud-->>Proxy: 7. Temporary credentials + Proxy->>Proxy: 8. Cache credentials + Proxy->>Store: 9. Forward request with
temporary credentials + Store-->>Proxy: 10. Response + Proxy-->>Client: 11. Response +``` + +### Configuration + +OIDC backend auth requires two environment variables: + +| Variable | Description | +|----------|-------------| +| `OIDC_PROVIDER_KEY` | PEM-encoded RSA private key for JWT signing | +| `OIDC_PROVIDER_ISSUER` | Publicly reachable URL (e.g., `https://data.source.coop`) | + +Generate an RSA key pair: + +```bash +openssl genrsa -out oidc-key.pem 2048 +``` + +Set the environment variables: + +```bash +export OIDC_PROVIDER_KEY=$(cat oidc-key.pem) +export OIDC_PROVIDER_ISSUER="https://data.source.coop" +``` + +Then configure buckets to use OIDC: + +```toml +[[buckets]] +name = "my-data" +backend_type = "s3" + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" +auth_type = "oidc" +oidc_role_arn = "arn:aws:iam::123456789012:role/DataProxyAccess" +``` + +### Discovery Endpoints + +When OIDC provider keys are configured, the proxy serves two well-known endpoints that cloud providers use to validate JWTs: + +**`GET /.well-known/openid-configuration`** +```json +{ + "issuer": "https://data.source.coop", + "jwks_uri": "https://data.source.coop/.well-known/jwks.json", + "response_types_supported": ["id_token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"] +} +``` + +**`GET /.well-known/jwks.json`** +```json +{ + "keys": [{ + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "kid": "proxy-key-1", + "n": "", + "e": "" + }] +} +``` + +::: warning +These endpoints must be publicly accessible. Cloud providers fetch them at JWT validation time to verify signatures. If they are behind a firewall or VPN, credential exchange will fail. +::: + +### The Exchange Flow in Detail + +When a request arrives for a bucket with `auth_type=oidc`: + +1. The `OidcBackendAuth` handler detects `auth_type=oidc` in the bucket's `backend_options` +2. It mints a short-lived JWT signed with the proxy's RSA private key: + - `iss`: the configured `OIDC_PROVIDER_ISSUER` + - `sub`: a connection identifier (from `oidc_subject` option, or a default) + - `aud`: the cloud provider's STS audience (e.g., `sts.amazonaws.com`) + - `exp`: short expiration (minutes) +3. The proxy sends the JWT to the cloud provider's STS endpoint along with the target IAM role ARN +4. The cloud provider fetches the proxy's JWKS, verifies the JWT signature, evaluates the role's trust policy, and returns temporary credentials +5. The proxy caches the credentials (keyed by role ARN) and injects them into the bucket config +6. The existing `build_object_store()` / `build_signer()` pipeline consumes the credentials normally + +On subsequent requests, cached credentials are reused until they expire. + +## Cloud Provider Setup + +### AWS S3 + +**Administrator setup:** + +1. **Register the OIDC provider** in your AWS account: + ```bash + aws iam create-open-id-connect-provider \ + --url https://data.source.coop \ + --client-id-list sts.amazonaws.com \ + --thumbprint-list + ``` + + ::: tip + To get the thumbprint, fetch the TLS certificate chain from your proxy's domain. AWS uses this to verify the HTTPS connection to the JWKS endpoint. + ::: + +2. **Create an IAM Role** with a trust policy that allows the proxy to assume it: + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/data.source.coop" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "data.source.coop:aud": "sts.amazonaws.com", + "data.source.coop:sub": "s3-proxy" + } + } + }] + } + ``` + +3. **Attach an S3 permission policy** to the role: + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::my-backend-bucket", + "arn:aws:s3:::my-backend-bucket/*" + ] + }] + } + ``` + +4. **Configure the bucket** in the proxy: + ```toml + [[buckets]] + name = "my-data" + backend_type = "s3" + + [buckets.backend_options] + endpoint = "https://s3.us-east-1.amazonaws.com" + bucket_name = "my-backend-bucket" + region = "us-east-1" + auth_type = "oidc" + oidc_role_arn = "arn:aws:iam::123456789012:role/DataProxyAccess" + ``` + +**At request time**, the proxy calls AWS STS `AssumeRoleWithWebIdentity` with the self-signed JWT. No AWS credentials are stored in the proxy configuration. + +### Azure Blob Storage + +::: info Planned +Azure OIDC backend auth is planned but not yet implemented. The proxy currently supports Azure with static credentials only. +::: + +**Planned setup:** + +1. Create an App Registration in Microsoft Entra ID +2. Add a Federated Identity Credential specifying the proxy's issuer URL and expected `sub` claim +3. Grant the app `Storage Blob Data Contributor` on the target storage account +4. The proxy would exchange its JWT for an Azure AD token via `client_credentials` grant with `jwt-bearer` assertion + +### Google Cloud Storage + +::: info Planned +GCS OIDC backend auth is planned but not yet implemented. The proxy currently supports GCS with static credentials only. +::: + +**Planned setup:** + +1. Create a Workload Identity Pool and OIDC Provider, specifying the proxy's issuer URL +2. Map the external identity to a GCP Service Account +3. Grant the service account GCS permissions +4. The proxy would use a two-step exchange: GCP STS token exchange, then `generateAccessToken` to impersonate the service account + +## Credential Caching + +When using OIDC backend auth, the proxy caches temporary credentials to avoid calling the cloud provider's STS on every request. Credentials are: + +- Keyed by the IAM role ARN +- Automatically refreshed when they expire +- Shared across concurrent requests to the same bucket + +This means the first request to an OIDC-backed bucket incurs a small latency cost for the credential exchange, but subsequent requests use cached credentials until they expire. + +## Choosing Between Static and OIDC + +| | Static Credentials | OIDC Backend Auth | +|---|---|---| +| **Setup complexity** | Low | Medium (IAM role + OIDC provider registration) | +| **Credential rotation** | Manual | Automatic (temporary credentials) | +| **Security** | Long-lived secrets in config | No long-lived secrets | +| **Cloud providers** | All (S3, Azure, GCS) | AWS S3 (Azure and GCS planned) | +| **Latency** | None | Small cost on first request (then cached) | diff --git a/docs/auth/index.md b/docs/auth/index.md new file mode 100644 index 0000000..213ed78 --- /dev/null +++ b/docs/auth/index.md @@ -0,0 +1,46 @@ +# Authentication + +The Source Data Proxy has two distinct authentication concerns: + +1. **Client authentication** — How clients prove their identity to the proxy +2. **Backend authentication** — How the proxy authenticates with backend object stores + +```mermaid +flowchart LR + Client["S3 Client"] + Proxy["Data Proxy"] + Backend["Object Store
(S3, Azure, GCS)"] + + Client -- "SigV4, STS/OIDC" --> Proxy + Proxy -- "Presigned URLs,
OIDC Exchange,
or Static Credentials" --> Backend +``` + +## Client Authentication + +Clients authenticate with the proxy using one of three methods: + +| Method | Use Case | How It Works | +|--------|----------|--------------| +| **Anonymous** | Public datasets | No credentials needed for GET/HEAD/LIST | +| **Long-lived access keys** | Service accounts, internal tools | Static `AccessKeyId`/`SecretAccessKey` with SigV4 signing | +| **OIDC/STS temporary credentials** | CI/CD, user sessions, federated identity | Exchange a JWT from an OIDC provider for scoped temporary credentials | + +The proxy verifies all signed requests using standard AWS Signature Version 4 (SigV4). Any S3-compatible client works without modification — just set the endpoint URL. + +The OIDC/STS flow is the recommended approach for most use cases. See [Client Auth Setup](./proxy-auth) for configuration details. + +## Backend Authentication + +The proxy authenticates with backend object stores using one of two methods: + +| Method | Use Case | How It Works | +|--------|----------|--------------| +| **Static credentials** | Simple setups | `access_key_id`/`secret_access_key` stored in the proxy config | +| **OIDC backend auth** | Production, credential-free | Proxy acts as its own OIDC provider, exchanges self-signed JWTs for cloud credentials | + +OIDC backend auth eliminates the need to store long-lived backend credentials. See [Backend Auth](./backend-auth) for details. + +## Related Topics + +- [Sealed Session Tokens](./sealed-tokens) — How temporary credentials are encrypted for stateless runtimes +- [User Guide: Authentication](/guide/authentication) — User-facing guide for obtaining credentials and using the CLI diff --git a/docs/auth/proxy-auth.md b/docs/auth/proxy-auth.md new file mode 100644 index 0000000..85c9ca1 --- /dev/null +++ b/docs/auth/proxy-auth.md @@ -0,0 +1,353 @@ +# Client Authentication Setup + +This page covers how to configure the proxy to authenticate incoming client requests. For the user-facing guide on obtaining credentials and using the CLI, see the [User Guide: Authentication](/guide/authentication). + +## Authentication Modes + +The proxy supports three authentication modes: + +| Mode | Config | Use Case | +|------|--------|----------| +| **Anonymous** | `anonymous_access = true` on a bucket | Public datasets, open data | +| **Long-lived access keys** | `[[credentials]]` entries | Service accounts, internal tools | +| **OIDC/STS temporary credentials** | `[[roles]]` with trust policies | CI/CD, user sessions, federated identity | + +## Anonymous Access + +Enable per-bucket: + +```toml +[[buckets]] +name = "public-data" +backend_type = "s3" +anonymous_access = true +``` + +Anonymous access only allows `GetObject`, `HeadObject`, and `ListBucket`. Write operations always require authentication. + +## Long-Lived Access Keys + +Static credentials are defined in the config. Each has an access key pair and scoped permissions: + +```toml +[[credentials]] +access_key_id = "AKPROXY00000EXAMPLE" +secret_access_key = "proxy/secret/key/EXAMPLE000000000000" +principal_name = "internal-dashboard" +created_at = "2024-01-15T00:00:00Z" +enabled = true + +[[credentials.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/production/"] +actions = ["get_object", "head_object"] +``` + +Clients sign requests using standard AWS SigV4. Any S3-compatible client works without modification. + +## OIDC/STS Temporary Credentials + +This is the recommended authentication method. Clients exchange a JWT from an OIDC-compatible identity provider for scoped, time-limited credentials via `AssumeRoleWithWebIdentity`. + +### How It Works + +```mermaid +sequenceDiagram + participant Client + participant OIDC as OIDC Provider
(Auth0, GitHub, etc.) + participant Proxy as Data Proxy + participant JWKS as Provider JWKS + + Client->>OIDC: 1. Authenticate + OIDC-->>Client: 2. JWT (id_token) + Client->>Proxy: 3. AssumeRoleWithWebIdentity
(JWT + RoleArn) + Proxy->>JWKS: 4. Fetch JWKS (cached) + JWKS-->>Proxy: 5. Public keys + Proxy->>Proxy: 6. Verify JWT signature (RS256) + Proxy->>Proxy: 7. Check trust policy
(issuer, audience, subject) + Proxy->>Proxy: 8. Mint temporary credentials
(seal into session token) + Proxy-->>Client: 9. AccessKeyId + SecretAccessKey
+ SessionToken + Expiration + Client->>Proxy: 10. S3 request with SigV4
(using temporary credentials) +``` + +### Verification Flow + +When a client calls `AssumeRoleWithWebIdentity`: + +1. The proxy decodes the JWT header to extract the `iss` (issuer) and `kid` (key ID) +2. The proxy verifies the issuer is trusted by the requested role +3. The proxy fetches the issuer's JWKS endpoint and verifies the JWT signature (RS256) +4. The proxy evaluates the trust policy: + - **Issuer**: must be in the role's `trusted_oidc_issuers` + - **Audience**: if `required_audience` is set on the role, the token's `aud` claim must match + - **Subject**: the token's `sub` claim must match at least one of the role's `subject_conditions` (supports `*` glob wildcards) +5. The proxy mints temporary credentials scoped to the role's `allowed_scopes` +6. If `SESSION_TOKEN_KEY` is configured, the credentials are AES-256-GCM encrypted into the session token (see [Sealed Session Tokens](./sealed-tokens)) +7. The proxy returns the credentials in an XML response matching the AWS STS format + +### STS Request Parameters + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `Action` | Yes | Must be `AssumeRoleWithWebIdentity` | +| `RoleArn` | Yes | The `role_id` of the role to assume | +| `WebIdentityToken` | Yes | The JWT from the OIDC provider | +| `DurationSeconds` | No | Session duration (900s minimum, capped by `max_session_duration_secs`) | + +### STS Response + +The response follows the standard AWS STS XML format: + +```xml + + + + STSPRXY... + ... + ... + 2024-01-15T01:00:00Z + + + github-actions-deployer/alice + github-actions-deployer + + + +``` + +## Integrating with OIDC Providers + +The proxy works with any OIDC-compliant identity provider that serves a JWKS endpoint and issues RS256-signed JWTs. You need: + +1. The provider's issuer URL (must serve `/.well-known/openid-configuration` with a `jwks_uri`) +2. The `sub` claim format for configuring `subject_conditions` +3. Optionally, the audience claim value for `required_audience` + +
+GitHub Actions — OIDC tokens for CI/CD workflows + +#### Role Configuration + +```toml +[[roles]] +role_id = "github-actions-deployer" +name = "GitHub Actions Deploy Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +required_audience = "sts.s3proxy.example.com" +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", + "repo:myorg/myapp:ref:refs/heads/release/*", +] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] +actions = ["get_object", "head_object", "put_object"] +``` + +#### Workflow Example + +```yaml +jobs: + deploy: + permissions: + id-token: write # Required for OIDC token + steps: + - name: Get OIDC token + id: oidc + run: | + TOKEN=$(curl -s \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.s3proxy.example.com" \ + | jq -r '.value') + echo "token=$TOKEN" >> $GITHUB_OUTPUT + + - name: Assume role via STS + run: | + CREDS=$(aws sts assume-role-with-web-identity \ + --role-arn github-actions-deployer \ + --web-identity-token ${{ steps.oidc.outputs.token }} \ + --endpoint-url https://s3proxy.example.com \ + --output json) + + echo "AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId')" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey')" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken')" >> $GITHUB_ENV + + - name: Upload to proxy + run: | + aws s3 cp ./bundle.tar.gz s3://deploy-bundles/releases/v1.2.3.tar.gz \ + --endpoint-url https://s3proxy.example.com +``` + +#### Key Details + +- **Issuer URL**: `https://token.actions.githubusercontent.com` +- **Subject format**: `repo:{owner}/{repo}:ref:{ref}` (e.g., `repo:myorg/myapp:ref:refs/heads/main`) +- **Audience**: configurable via the `&audience=` parameter in the token request URL +- The `id-token: write` permission is required in the workflow + +
+ +
+Auth0 — OAuth2/OIDC identity platform + +#### Auth0 Setup + +1. Create an Application (Regular Web Application or SPA) in your Auth0 dashboard +2. Note your Auth0 domain — this is the issuer URL + +#### Role Configuration + +```toml +[[roles]] +role_id = "auth0-user" +name = "Auth0 User" +trusted_oidc_issuers = ["https://your-tenant.auth0.com/"] +required_audience = "https://s3proxy.example.com" +subject_conditions = ["*"] # Or restrict by user ID patterns +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +#### Key Details + +- **Issuer URL**: `https://your-tenant.auth0.com/` (trailing slash required) +- **Subject claim**: Auth0 user ID (e.g., `auth0|507f1f77bcf86cd799439011`) +- **Audience**: set when requesting the token via the `audience` parameter + +
+ +
+Keycloak — Open-source identity and access management + +#### Keycloak Setup + +1. Create a Realm and a Client in your Keycloak admin console +2. Set the client's Access Type to `public` or `confidential` as needed +3. Enable "Standard Flow" (Authorization Code) + +#### Role Configuration + +```toml +[[roles]] +role_id = "keycloak-user" +name = "Keycloak User" +trusted_oidc_issuers = ["https://keycloak.example.com/realms/myrealm"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +#### Key Details + +- **Issuer URL**: `https://keycloak.example.com/realms/{realm-name}` +- **Subject claim**: Keycloak user UUID +- **JWKS**: served at `{issuer}/protocol/openid-connect/certs` + +
+ +
+AWS Cognito — AWS-managed identity service + +#### Cognito Setup + +1. Create a User Pool in the AWS Cognito console +2. Create an App Client (no client secret for public clients) +3. Configure the Hosted UI or use the Cognito SDK for authentication + +#### Role Configuration + +```toml +[[roles]] +role_id = "cognito-user" +name = "Cognito User" +trusted_oidc_issuers = ["https://cognito-idp.us-east-1.amazonaws.com/us-east-1_EXAMPLE"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +#### Key Details + +- **Issuer URL**: `https://cognito-idp.{region}.amazonaws.com/{user-pool-id}` +- **Subject claim**: Cognito user UUID +- **Audience**: the App Client ID (set `required_audience` to match) + +
+ +
+Ory / Ory Network — Open-source OAuth2/OIDC infrastructure + +#### Ory Setup + +1. Create an OAuth2 client as a **public client** (no client secret) +2. Set the grant type to Authorization Code with PKCE +3. Register `http://127.0.0.1/callback` as a redirect URI (any port is allowed per RFC 8252) +4. Set allowed scopes to include `openid` + +#### Role Configuration + +```toml +[[roles]] +role_id = "ory-user" +name = "Ory User" +trusted_oidc_issuers = ["https://your-project.projects.oryapis.com"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +#### Key Details + +- **Issuer URL**: `https://your-project.projects.oryapis.com` (Ory Network) or your self-hosted Hydra URL +- **Subject claim**: Ory identity UUID +- The CLI (`source-coop login`) works well with Ory since Ory follows RFC 8252 for loopback redirect URIs + +
+ +## Template Variables in Scopes + +Role scopes support `{claim_name}` template variables that are resolved from the authenticated user's JWT claims when credentials are minted. This enables per-user access without creating a separate role for each user. + +```toml +[[roles]] +role_id = "source-coop-user" +trusted_oidc_issuers = ["https://auth.source.coop"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +# Each user gets access to a bucket matching their OIDC subject +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +A user with `sub = "alice"` receives credentials scoped to `bucket = "alice"`. Any string claim from the JWT can be referenced — `{email}`, `{org}`, etc. Missing or non-string claims resolve to an empty string, which safely fails authorization. + +You can also use template variables in prefixes for more granular access: + +```toml +[[roles.allowed_scopes]] +bucket = "shared-data" +prefixes = ["{org}/"] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` diff --git a/docs/auth/sealed-tokens.md b/docs/auth/sealed-tokens.md new file mode 100644 index 0000000..2b35ba7 --- /dev/null +++ b/docs/auth/sealed-tokens.md @@ -0,0 +1,67 @@ +# Sealed Session Tokens + +When the proxy mints temporary credentials via STS, it needs a way to recognize those credentials on subsequent requests. Sealed session tokens solve this by encrypting the full credential set into the session token itself — no server-side storage required. + +## Why Sealed Tokens? + +Traditional credential stores keep a mapping from access key ID to credentials on the server. This requires either a database or in-memory state, which is impractical for stateless runtimes like Cloudflare Workers. + +Sealed tokens take a different approach: the credentials are encrypted and placed directly inside the session token that the client sends with every request. The proxy decrypts the token on each request to recover the credentials. + +## How It Works + +### Minting (seal) + +When `AssumeRoleWithWebIdentity` mints temporary credentials: + +1. The full `TemporaryCredentials` struct is serialized to JSON +2. A random 12-byte nonce is generated +3. The JSON is encrypted using AES-256-GCM with the nonce +4. The result is encoded as `base64url(nonce[12] || ciphertext + tag)` +5. This encoded string becomes the `SessionToken` returned to the client + +### Verifying (unseal) + +When a request arrives with an `x-amz-security-token` header: + +1. The proxy base64url-decodes the session token +2. It extracts the nonce (first 12 bytes) and ciphertext (remainder) +3. It decrypts using AES-256-GCM with the configured key +4. The JSON is deserialized back to `TemporaryCredentials` +5. The proxy checks that the credentials haven't expired +6. The proxy verifies the request's SigV4 signature against the decrypted secret key + +If the token doesn't look like a sealed token (e.g., not valid base64url), the proxy falls back to looking up credentials from the config provider. + +## Configuration + +Set the `SESSION_TOKEN_KEY` environment variable to a base64-encoded 32-byte key: + +```bash +# Generate a key +openssl rand -base64 32 + +# Set it +export SESSION_TOKEN_KEY="" +``` + +This key must be the same across all instances of the proxy. If you rotate the key, all existing session tokens become invalid — clients will need to re-authenticate. + +::: warning +`SESSION_TOKEN_KEY` is required for the Cloudflare Workers runtime. Without it, temporary credentials from STS cannot be verified on subsequent requests. +::: + +## Scope Behavior + +Access scopes are sealed into the token at mint time. This means: + +- Changing a role's `allowed_scopes` in the config only affects newly minted credentials +- Existing session tokens continue to use the scopes they were minted with until they expire +- There is no way to revoke a sealed token short of rotating the encryption key (which invalidates all tokens) + +## Security Properties + +- **Confidentiality**: AES-256-GCM encryption prevents clients from reading or modifying the sealed credentials +- **Integrity**: The GCM authentication tag detects any tampering with the ciphertext +- **Replay protection**: Each token has a random nonce; however, tokens are valid until their expiration time +- **Constant-time comparison**: The access key ID verification uses constant-time comparison to prevent timing attacks diff --git a/docs/configuration/buckets.md b/docs/configuration/buckets.md new file mode 100644 index 0000000..86edd21 --- /dev/null +++ b/docs/configuration/buckets.md @@ -0,0 +1,136 @@ +# Buckets + +Buckets define the virtual namespaces that clients interact with. Each bucket maps a client-visible name to a backend object store. + +## Configuration + +```toml +[[buckets]] +name = "my-data" +backend_type = "s3" +backend_prefix = "v2" +anonymous_access = false +allowed_roles = ["github-actions-deployer"] + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" +access_key_id = "AKIAIOSFODNN7EXAMPLE" +secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +``` + +## Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Client-visible bucket name | +| `backend_type` | string | Yes | Backend provider: `"s3"`, `"az"`, or `"gcs"` | +| `backend_prefix` | string | No | Prefix prepended to keys when forwarding to the backend | +| `anonymous_access` | bool | No | Allow GET/HEAD/LIST without authentication (default: `false`) | +| `allowed_roles` | string[] | No | Role IDs that can be assumed for this bucket | +| `backend_options` | map | Yes | Provider-specific configuration (see below) | + +## Backend Options by Provider + +### S3 / MinIO / R2 + +```toml +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" +access_key_id = "AKIA..." +secret_access_key = "..." +``` + +| Option | Required | Description | +|--------|----------|-------------| +| `endpoint` | Yes | S3 endpoint URL | +| `bucket_name` | Yes | Backend bucket name | +| `region` | Yes | AWS region | +| `access_key_id` | No | AWS access key (omit for anonymous or OIDC) | +| `secret_access_key` | No | AWS secret key | +| `skip_signature` | No | Set to `"true"` for unsigned requests | + +### Azure Blob Storage + +::: info +Requires the `azure` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. +::: + +```toml +[buckets.backend_options] +account_name = "mystorageaccount" +container_name = "my-container" +access_key = "..." +``` + +| Option | Required | Description | +|--------|----------|-------------| +| `account_name` | Yes | Azure storage account name | +| `container_name` | Yes | Blob container name | +| `access_key` | No | Storage account access key | +| `skip_signature` | No | Set to `"true"` for anonymous access | + +### Google Cloud Storage + +::: info +Requires the `gcp` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. +::: + +```toml +[buckets.backend_options] +bucket_name = "my-gcs-bucket" +service_account_key = '{"type": "service_account", ...}' +``` + +| Option | Required | Description | +|--------|----------|-------------| +| `bucket_name` | Yes | GCS bucket name | +| `service_account_key` | No | JSON service account key | +| `skip_signature` | No | Set to `"true"` for anonymous access | + +### OIDC Backend Auth Options + +For any backend type, you can use OIDC-based credential resolution instead of static credentials: + +```toml +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" +auth_type = "oidc" +oidc_role_arn = "arn:aws:iam::123456789012:role/ProxyRole" +oidc_subject = "my-connection" # optional, defaults to "s3-proxy" +``` + +See [Authenticating with Backends](/auth/backend-auth) for setup details. + +## Backend Prefix + +The `backend_prefix` field transparently prepends a path prefix to all keys when forwarding requests to the backend. Clients don't see this prefix. + +```toml +[[buckets]] +name = "ml-artifacts" +backend_prefix = "v2" + +[buckets.backend_options] +bucket_name = "ml-pipeline-artifacts" +``` + +With this configuration: +- Client requests `GET /ml-artifacts/models/latest.pt` +- Proxy forwards to backend key `v2/models/latest.pt` in `ml-pipeline-artifacts` +- LIST responses have the prefix stripped so clients see `models/latest.pt` + +## Anonymous Access + +Setting `anonymous_access = true` allows unauthenticated GET, HEAD, and LIST requests. Write operations (PUT, DELETE, multipart) always require authentication regardless of this setting. + +```toml +[[buckets]] +name = "public-data" +anonymous_access = true +``` diff --git a/docs/configuration/credentials.md b/docs/configuration/credentials.md new file mode 100644 index 0000000..f9cd425 --- /dev/null +++ b/docs/configuration/credentials.md @@ -0,0 +1,66 @@ +# Credentials + +Long-lived credentials are static access key pairs stored in the proxy configuration. They work like standard AWS IAM access keys — clients sign requests using SigV4 with the access key ID and secret access key. + +## Configuration + +```toml +[[credentials]] +access_key_id = "AKPROXY00000EXAMPLE" +secret_access_key = "proxy/secret/key/EXAMPLE000000000000" +principal_name = "internal-dashboard" +created_at = "2024-01-15T00:00:00Z" +enabled = true + +[[credentials.allowed_scopes]] +bucket = "public-data" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] + +[[credentials.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/production/"] +actions = ["get_object", "head_object"] +``` + +## Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `access_key_id` | string | Yes | Access key identifier | +| `secret_access_key` | string | Yes | Secret key for SigV4 signing | +| `principal_name` | string | Yes | Human-readable name for the credential holder | +| `created_at` | datetime | Yes | When the credential was created (ISO 8601) | +| `expires_at` | datetime | No | When the credential expires (omit for no expiration) | +| `enabled` | bool | Yes | Whether the credential is active | +| `allowed_scopes` | AccessScope[] | Yes | Buckets, prefixes, and actions granted | + +## Access Scopes + +Scopes work identically to [role scopes](./roles#access-scopes) — each scope specifies a bucket, optional prefix restrictions, and allowed actions. + +## When to Use Long-Lived Credentials + +Long-lived credentials are appropriate for: + +- **Service accounts** that need persistent access without OIDC +- **Internal tools** where token exchange adds unnecessary complexity +- **Development and testing** environments +- **Environments without an OIDC provider** + +For CI/CD workflows and user-facing applications, prefer [OIDC/STS temporary credentials](/auth/proxy-auth#oidcsts-temporary-credentials) for better security (automatic expiration, no stored secrets). + +## Disabling Credentials + +Set `enabled = false` to immediately revoke access without removing the credential from config: + +```toml +[[credentials]] +access_key_id = "AKPROXY00000REVOKED" +secret_access_key = "..." +principal_name = "old-service" +created_at = "2023-01-01T00:00:00Z" +enabled = false +``` + +Disabled credentials return `AccessDenied` for any request. diff --git a/docs/configuration/index.md b/docs/configuration/index.md new file mode 100644 index 0000000..293ef76 --- /dev/null +++ b/docs/configuration/index.md @@ -0,0 +1,68 @@ +# Configuration + +The proxy configuration defines three things: + +1. **[Buckets](./buckets)** — Virtual buckets that map client-visible names to backend object stores +2. **[Roles](./roles)** — Trust policies for OIDC token exchange via `AssumeRoleWithWebIdentity` +3. **[Credentials](./credentials)** — Long-lived access keys for service accounts and internal tools + +```mermaid +flowchart TD + Config["Proxy Configuration"] + Config --> Buckets["Buckets
(virtual names → backends)"] + Config --> Roles["Roles
(OIDC trust policies)"] + Config --> Creds["Credentials
(static access keys)"] + + Roles -- "allowed_scopes" --> Buckets + Creds -- "allowed_scopes" --> Buckets +``` + +## Config Format + +The server runtime uses TOML: + +```toml +[[buckets]] +name = "public-data" +backend_type = "s3" +anonymous_access = true + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-public-assets" +region = "us-east-1" +``` + +The CF Workers runtime uses JSON (as an environment variable or `wrangler.toml` object): + +```json +{ + "buckets": [{ + "name": "public-data", + "backend_type": "s3", + "anonymous_access": true, + "backend_options": { + "endpoint": "https://s3.us-east-1.amazonaws.com", + "bucket_name": "my-public-assets", + "region": "us-east-1" + } + }] +} +``` + +## Config Providers + +The proxy can load configuration from multiple backends. See [Config Providers](./providers/) for details. + +| Provider | Feature Flag | Use Case | +|----------|-------------|----------| +| [Static File](./providers/static-file) | (always available) | Simple deployments, baked-in config | +| [HTTP API](./providers/http) | `config-http` | Centralized config service | +| [DynamoDB](./providers/dynamodb) | `config-dynamodb` | AWS-native infrastructure | +| [PostgreSQL](./providers/postgres) | `config-postgres` | Database-backed config | + +All providers can be wrapped with a [cache](./providers/cached) for performance. + +## Full Example + +See the [annotated config example](/reference/config-example) for a complete configuration file with all options documented. diff --git a/docs/configuration/providers/cached.md b/docs/configuration/providers/cached.md new file mode 100644 index 0000000..a1597db --- /dev/null +++ b/docs/configuration/providers/cached.md @@ -0,0 +1,42 @@ +# Caching + +Wrap any config provider with `CachedProvider` to add in-memory TTL-based caching. This is recommended for all network-backed providers (HTTP, DynamoDB, PostgreSQL). + +## Usage + +```rust +use source_coop_core::config::cached::CachedProvider; +use std::time::Duration; + +let base = HttpProvider::new("https://config-api.internal".into(), None); +let provider = CachedProvider::new(base, Duration::from_secs(300)); +``` + +The first call hits the underlying provider; subsequent calls within the TTL return cached data. + +## Cache Behavior + +- **Thread-safe**: Uses `RwLock` internally, safe for concurrent access +- **Lazy eviction**: Expired entries are evicted on access, not proactively +- **Per-entity caching**: Each bucket, role, and credential is cached independently +- **Temporary credentials bypass**: Credential store/get operations for temporary credentials are not cached + +## Manual Invalidation + +```rust +// Invalidate everything +provider.invalidate_all(); + +// Invalidate a specific bucket +provider.invalidate_bucket("my-bucket"); +``` + +## Recommended TTLs + +| Provider | Suggested TTL | Rationale | +|----------|--------------|-----------| +| HTTP API | 60–300s | Balance between freshness and API load | +| DynamoDB | 60–300s | Reduce read capacity costs | +| PostgreSQL | 30–120s | Reduce query load | + +The server runtime's binary uses a 60-second TTL by default when wrapping the static file provider. diff --git a/docs/configuration/providers/dynamodb.md b/docs/configuration/providers/dynamodb.md new file mode 100644 index 0000000..59df1d0 --- /dev/null +++ b/docs/configuration/providers/dynamodb.md @@ -0,0 +1,33 @@ +# DynamoDB Provider + +The DynamoDB provider stores configuration in a single DynamoDB table using a PK/SK (partition key / sort key) design pattern. + +## Feature Flag + +```bash +cargo build -p source-coop-server --features source-coop-core/config-dynamodb +``` + +## Usage + +```rust +use source_coop_core::config::dynamodb::DynamoDbProvider; + +let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; +let client = aws_sdk_dynamodb::Client::new(&aws_config); +let provider = DynamoDbProvider::new(client, "source-coop-proxy-config".to_string()); +``` + +## Table Design + +The provider uses a single-table design with partition key (`PK`) and sort key (`SK`) attributes. + +## When to Use + +- AWS-native infrastructure +- Serverless deployments where a database server isn't practical +- High-availability requirements (DynamoDB's built-in replication) + +::: tip +Wrap the DynamoDB provider with [CachedProvider](./cached) to reduce read costs and latency. +::: diff --git a/docs/configuration/providers/http.md b/docs/configuration/providers/http.md new file mode 100644 index 0000000..9451173 --- /dev/null +++ b/docs/configuration/providers/http.md @@ -0,0 +1,39 @@ +# HTTP API Provider + +The HTTP provider fetches configuration from a centralized REST API. Useful when you have a control plane service that manages proxy configuration. + +## Feature Flag + +```bash +cargo build -p source-coop-server --features source-coop-core/config-http +``` + +## Usage + +```rust +use source_coop_core::config::http::HttpProvider; + +let provider = HttpProvider::new( + "https://config-api.internal:8080".to_string(), + Some("Bearer my-api-token".to_string()), +); +``` + +## Expected API Endpoints + +The HTTP provider expects a REST API with these endpoints: + +| Endpoint | Method | Returns | +|----------|--------|---------| +| `/buckets` | GET | `Vec` | +| `/buckets/{name}` | GET | `Option` | +| `/roles/{id}` | GET | `Option` | +| `/credentials/{access_key_id}` | GET | `Option` | + +All responses should be JSON-encoded. Missing resources should return `null` or a 404 status. + +## When to Use + +- Centralized config management across multiple proxy instances +- Dynamic configuration that changes without proxy restarts (when combined with [caching](./cached)) +- Integration with a custom control plane or admin dashboard diff --git a/docs/configuration/providers/index.md b/docs/configuration/providers/index.md new file mode 100644 index 0000000..b40deef --- /dev/null +++ b/docs/configuration/providers/index.md @@ -0,0 +1,57 @@ +# Config Providers + +The proxy loads its configuration (buckets, roles, credentials) through the `ConfigProvider` trait. Multiple backends are available, selectable at build time via feature flags. + +## ConfigProvider Trait + +```rust +pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { + async fn list_buckets(&self) -> Result, ProxyError>; + async fn get_bucket(&self, name: &str) -> Result, ProxyError>; + async fn get_role(&self, role_id: &str) -> Result, ProxyError>; + async fn get_credential(&self, access_key_id: &str) + -> Result, ProxyError>; +} +``` + +## Available Providers + +| Provider | Feature Flag | Best For | +|----------|-------------|----------| +| [Static File](./static-file) | (always available) | Simple deployments, single-file config | +| [HTTP API](./http) | `config-http` | Centralized config service, control planes | +| [DynamoDB](./dynamodb) | `config-dynamodb` | AWS-native infrastructure | +| [PostgreSQL](./postgres) | `config-postgres` | Database-backed config | + +All providers can be wrapped with [CachedProvider](./cached) for in-memory caching with TTL-based expiration. + +## Implementing a Custom Provider + +Implement the `ConfigProvider` trait and wrap it in `DefaultResolver` to get standard S3 proxy behavior: + +```rust +use source_coop_core::config::ConfigProvider; +use source_coop_core::error::ProxyError; +use source_coop_core::types::*; + +#[derive(Clone)] +struct MyProvider { /* ... */ } + +impl ConfigProvider for MyProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + todo!() + } + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + todo!() + } + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + todo!() + } + async fn get_credential(&self, access_key_id: &str) + -> Result, ProxyError> { + todo!() + } +} +``` + +See [Custom Config Provider](/extending/custom-provider) for a full guide. diff --git a/docs/configuration/providers/postgres.md b/docs/configuration/providers/postgres.md new file mode 100644 index 0000000..f7516de --- /dev/null +++ b/docs/configuration/providers/postgres.md @@ -0,0 +1,28 @@ +# PostgreSQL Provider + +The PostgreSQL provider stores configuration in a PostgreSQL database using sqlx. + +## Feature Flag + +```bash +cargo build -p source-coop-server --features source-coop-core/config-postgres +``` + +## Usage + +```rust +use source_coop_core::config::postgres::PostgresProvider; + +let pool = sqlx::PgPool::connect("postgres://localhost/s3proxy").await?; +let provider = PostgresProvider::new(pool); +``` + +## When to Use + +- Existing PostgreSQL infrastructure +- Relational data management preferences +- Complex queries or joins with other application data + +::: tip +Wrap the PostgreSQL provider with [CachedProvider](./cached) to reduce query load and latency. +::: diff --git a/docs/configuration/providers/static-file.md b/docs/configuration/providers/static-file.md new file mode 100644 index 0000000..475be90 --- /dev/null +++ b/docs/configuration/providers/static-file.md @@ -0,0 +1,86 @@ +# Static File Provider + +The static file provider loads configuration from a TOML or JSON file at startup. No feature flags required — it's always available. + +## Usage + +```rust +use source_coop_core::config::static_file::StaticProvider; + +// From a TOML file +let provider = StaticProvider::from_file("config.toml")?; + +// From a TOML string +let provider = StaticProvider::from_toml(include_str!("../config.toml"))?; + +// From a JSON string (useful for CF Workers env vars) +let provider = StaticProvider::from_json(&json_string)?; +``` + +## Config Format + +### TOML + +```toml +[[buckets]] +name = "my-data" +backend_type = "s3" +anonymous_access = true + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-backend-bucket" +region = "us-east-1" + +[[roles]] +role_id = "my-role" +name = "My Role" +trusted_oidc_issuers = ["https://auth.example.com"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "my-data" +prefixes = [] +actions = ["get_object", "head_object"] + +[[credentials]] +access_key_id = "AKEXAMPLE" +secret_access_key = "secret" +principal_name = "service" +created_at = "2024-01-01T00:00:00Z" +enabled = true + +[[credentials.allowed_scopes]] +bucket = "my-data" +prefixes = [] +actions = ["get_object"] +``` + +### JSON + +```json +{ + "buckets": [{ + "name": "my-data", + "backend_type": "s3", + "anonymous_access": true, + "backend_options": { + "endpoint": "https://s3.us-east-1.amazonaws.com", + "bucket_name": "my-backend-bucket", + "region": "us-east-1" + } + }], + "roles": [], + "credentials": [] +} +``` + +## When to Use + +- Simple deployments with a single config file +- Baked-in configuration (e.g., compiled into the binary with `include_str!`) +- Cloudflare Workers (JSON via `PROXY_CONFIG` env var) +- Development and testing + +For dynamic configuration that changes without redeployment, consider [HTTP](./http), [DynamoDB](./dynamodb), or [PostgreSQL](./postgres) providers. diff --git a/docs/configuration/roles.md b/docs/configuration/roles.md new file mode 100644 index 0000000..4f5454d --- /dev/null +++ b/docs/configuration/roles.md @@ -0,0 +1,149 @@ +# Roles + +Roles define trust policies for OIDC token exchange via `AssumeRoleWithWebIdentity`. Each role specifies which identity providers to trust, what subject constraints to enforce, and what access scopes to grant. + +## Configuration + +```toml +[[roles]] +role_id = "github-actions-deployer" +name = "GitHub Actions Deploy Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +required_audience = "sts.s3proxy.example.com" +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", + "repo:myorg/infrastructure:*", +] +max_session_duration_secs = 3600 + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] +actions = ["get_object", "head_object", "put_object"] + +[[roles.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/", "datasets/"] +actions = ["get_object", "head_object"] +``` + +## Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `role_id` | string | Yes | Identifier used as the `RoleArn` in STS requests | +| `name` | string | Yes | Human-readable display name | +| `trusted_oidc_issuers` | string[] | Yes | OIDC provider URLs whose tokens are accepted | +| `required_audience` | string | No | If set, the token's `aud` claim must match | +| `subject_conditions` | string[] | Yes | Glob patterns matched against the `sub` claim | +| `max_session_duration_secs` | integer | Yes | Maximum session lifetime (minimum 900s) | +| `allowed_scopes` | AccessScope[] | Yes | Buckets, prefixes, and actions granted | + +## Trust Policy Evaluation + +When a client calls `AssumeRoleWithWebIdentity`, the proxy evaluates the JWT against the role's trust policy in this order: + +1. **Issuer** — The JWT's `iss` claim must match one of `trusted_oidc_issuers` +2. **Algorithm** — Only RS256 is supported +3. **Signature** — Verified against the issuer's JWKS (fetched and cached) +4. **Audience** — If `required_audience` is set, the JWT's `aud` claim must match +5. **Subject** — The JWT's `sub` claim must match at least one `subject_conditions` pattern + +If any check fails, the STS request returns an error. + +## Subject Conditions + +Subject conditions use glob-style matching where `*` matches any sequence of characters: + +```toml +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", # Exact match + "repo:myorg/myapp:ref:refs/heads/release/*", # Prefix match + "repo:myorg/*", # Any repo in the org + "*", # Any subject +] +``` + +The `sub` claim only needs to match one of the patterns. + +## Access Scopes + +Each scope grants access to a specific bucket with optional prefix and action restrictions: + +```toml +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = ["releases/", "staging/"] +actions = ["get_object", "head_object", "put_object"] +``` + +| Field | Type | Description | +|-------|------|-------------| +| `bucket` | string | Virtual bucket name (or template variable) | +| `prefixes` | string[] | Allowed key prefixes (empty = full bucket access) | +| `actions` | string[] | Allowed S3 operations | + +### Available Actions + +| Action | S3 Operation | +|--------|-------------| +| `get_object` | GET (download) | +| `head_object` | HEAD (metadata) | +| `put_object` | PUT (upload) | +| `delete_object` | DELETE | +| `list_bucket` | LIST (list objects) | +| `create_multipart_upload` | POST (initiate multipart) | +| `upload_part` | PUT with partNumber (upload part) | +| `complete_multipart_upload` | POST with uploadId (complete multipart) | +| `abort_multipart_upload` | DELETE with uploadId (abort multipart) | + +### Prefix Matching + +Prefix matching follows these rules: + +- If the prefix ends with `/` or is empty: the key must start with the prefix +- Otherwise: the key must equal the prefix exactly, or start with the prefix followed by `/` + +This prevents a prefix like `data` from accidentally matching `data-private/secret.txt`. The prefix `data/` would only match keys under the `data/` directory. + +## Template Variables + +Scope `bucket` and `prefixes` values support `{claim_name}` template variables that are resolved from the JWT claims at credential mint time: + +```toml +[[roles]] +role_id = "source-coop-user" +trusted_oidc_issuers = ["https://auth.source.coop"] +subject_conditions = ["*"] +max_session_duration_secs = 3600 + +# Each user gets access to a bucket matching their subject claim +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] +``` + +A user with `sub = "alice"` receives credentials scoped to `bucket = "alice"`. Any string claim from the JWT can be referenced — `{email}`, `{org}`, etc. + +Missing or non-string claims resolve to an empty string, which safely fails authorization. + +### Examples + +**Per-user bucket access:** +```toml +bucket = "{sub}" +``` + +**Organization-scoped prefix:** +```toml +bucket = "shared-data" +prefixes = ["{org}/"] +``` + +**Read-only access to all buckets:** +```toml +bucket = "*" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] +``` diff --git a/docs/deployment/cloudflare-workers.md b/docs/deployment/cloudflare-workers.md new file mode 100644 index 0000000..8499468 --- /dev/null +++ b/docs/deployment/cloudflare-workers.md @@ -0,0 +1,95 @@ +# Cloudflare Workers + +The CF Workers runtime deploys the proxy to Cloudflare's edge network. It compiles to WASM and runs in the Workers V8 environment. + +## Limitations + +- **S3 backends only** — Azure and GCS are not supported on WASM +- **Static or API config only** — DynamoDB and Postgres providers require Tokio, which is unavailable +- **`SESSION_TOKEN_KEY` required** — Workers are stateless, so sealed tokens are the only way to persist temporary credentials + +## Configuration + +### `wrangler.toml` + +```toml +name = "source-coop-proxy" +main = "build/worker/shim.mjs" +compatibility_date = "2024-01-01" + +[build] +command = "cargo install worker-build && worker-build --release" + +[vars] +VIRTUAL_HOST_DOMAIN = "s3.example.com" + +[vars.PROXY_CONFIG] +buckets = [ + { name = "public-data", backend_type = "s3", anonymous_access = true, backend_options = { endpoint = "https://s3.us-east-1.amazonaws.com", bucket_name = "my-bucket", region = "us-east-1" } } +] +roles = [] +credentials = [] +``` + +`PROXY_CONFIG` can be either: +- A JSON string (via `wrangler secret put PROXY_CONFIG`) +- A JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`, as shown above) + +### Secrets + +Set sensitive values as secrets: + +```bash +wrangler secret put SESSION_TOKEN_KEY +wrangler secret put OIDC_PROVIDER_KEY +``` + +### Environment Variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `PROXY_CONFIG` | Yes | JSON config (buckets, roles, credentials) | +| `VIRTUAL_HOST_DOMAIN` | No | Domain for virtual-hosted requests | +| `SESSION_TOKEN_KEY` | For STS | Base64-encoded 32-byte AES-256-GCM key | +| `OIDC_PROVIDER_KEY` | For OIDC backend auth | PEM-encoded RSA private key | +| `OIDC_PROVIDER_ISSUER` | For OIDC backend auth | Public URL for JWKS discovery | + +## Building + +```bash +# Check +cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown + +# Build (via Wrangler) +cd crates/runtimes/cf-workers +npx wrangler build +``` + +::: warning +Always use `--target wasm32-unknown-unknown` when checking or building the CF Workers crate. It is excluded from the workspace `default-members` because WASM types won't compile on native targets. +::: + +## Development + +```bash +cd crates/runtimes/cf-workers +npx wrangler dev +``` + +This starts a local dev server on port `8787`. + +## Deploying + +```bash +cd crates/runtimes/cf-workers +npx wrangler deploy +``` + +## Source Cooperative Mode + +When `SOURCE_API_URL` is set, the Workers runtime uses `SourceCoopResolver` instead of `DefaultResolver`. This mode: +- Resolves backends dynamically from the Source Cooperative API +- Maps URLs as `/{account_id}/{repo_id}/{key}` instead of `/{bucket}/{key}` +- Handles authorization via the Source Cooperative API permissions endpoint + +This is specific to Source Cooperative deployments and is not needed for standalone proxy use. diff --git a/docs/deployment/index.md b/docs/deployment/index.md new file mode 100644 index 0000000..62e9b89 --- /dev/null +++ b/docs/deployment/index.md @@ -0,0 +1,13 @@ +# Deployment + +The proxy can be deployed in two ways: + +| | [Server Runtime](./server) | [Cloudflare Workers](./cloudflare-workers) | +|---|---|---| +| **Best for** | Container environments (ECS, K8s, Docker) | Edge deployments, low-latency global access | +| **Backends** | S3, Azure, GCS | S3 only | +| **Scaling** | Horizontal (multiple instances) | Automatic (Cloudflare edge) | +| **Config** | TOML file + env vars | Env vars (JSON) + Wrangler secrets | +| **Complexity** | Standard ops (containers, load balancers) | Managed (no infrastructure to operate) | + +Both runtimes use the same core logic and support the same authentication flows. Choose based on your infrastructure preferences and backend requirements. diff --git a/docs/deployment/server.md b/docs/deployment/server.md new file mode 100644 index 0000000..ee5df5f --- /dev/null +++ b/docs/deployment/server.md @@ -0,0 +1,81 @@ +# Server Runtime + +The server runtime uses Tokio and Hyper to run as a native HTTP server. It supports all backend providers (S3, Azure, GCS) and all config providers. + +## Building + +```bash +# Default build (S3 + Azure + GCS backends) +cargo build --release -p source-coop-server + +# With additional config providers +cargo build --release -p source-coop-server \ + --features source-coop-core/config-dynamodb \ + --features source-coop-core/config-postgres +``` + +The binary is located at `target/release/source-coop-proxy`. + +## Running + +```bash +./target/release/source-coop-proxy \ + --config config.toml \ + --listen 0.0.0.0:8080 +``` + +### CLI Arguments + +| Flag | Default | Description | +|------|---------|-------------| +| `--config` | (required) | Path to the TOML config file | +| `--listen` | `0.0.0.0:8080` | Address and port to listen on | +| `--domain` | (none) | Domain for virtual-hosted-style requests (e.g., `s3.example.com`) | +| `--sts-config` | (none) | Optional separate TOML file for STS roles/credentials | + +### Environment Variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `SESSION_TOKEN_KEY` | For STS | Base64-encoded 32-byte AES-256-GCM key for sealed tokens | +| `OIDC_PROVIDER_KEY` | For OIDC backend auth | PEM-encoded RSA private key | +| `OIDC_PROVIDER_ISSUER` | For OIDC backend auth | Publicly reachable URL for JWKS discovery | +| `RUST_LOG` | No | Logging level (default: `source_coop=info`) | + +Generate a session token key: + +```bash +export SESSION_TOKEN_KEY=$(openssl rand -base64 32) +``` + +## Docker + +```bash +# Build +docker build -t source-coop-proxy . + +# Run +docker run \ + -v ./config.toml:/etc/source-coop-proxy/config.toml \ + -p 8080:8080 \ + -e SESSION_TOKEN_KEY="$SESSION_TOKEN_KEY" \ + source-coop-proxy +``` + +## Config Caching + +The server binary wraps the config provider with `CachedProvider` (60-second TTL). Config changes from network-backed providers (HTTP, DynamoDB, Postgres) are picked up within 60 seconds without restarting the proxy. + +For static file configs, changes require a restart. + +## Virtual-Hosted Style + +To support virtual-hosted-style requests (`bucket.s3.example.com/key`), use the `--domain` flag: + +```bash +./source-coop-proxy --config config.toml --domain s3.example.com +``` + +Configure DNS so that `*.s3.example.com` resolves to the proxy. The proxy extracts the bucket name from the `Host` header. + +Without `--domain`, only path-style requests are supported (`/bucket/key`). diff --git a/docs/extending/custom-backend.md b/docs/extending/custom-backend.md new file mode 100644 index 0000000..b38a606 --- /dev/null +++ b/docs/extending/custom-backend.md @@ -0,0 +1,120 @@ +# Custom Backend + +The `ProxyBackend` trait abstracts runtime-specific I/O. Implement it when deploying to a platform that's neither a standard server nor Cloudflare Workers. + +## The Trait + +```rust +use source_coop_core::backend::ProxyBackend; +use source_coop_core::types::BucketConfig; +use source_coop_core::error::ProxyError; +use object_store::{ObjectStore, signer::Signer}; +use std::sync::Arc; + +pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { + /// Create an ObjectStore for LIST operations + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; + + /// Create a Signer for presigned URL generation (GET/HEAD/PUT/DELETE) + fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError>; + + /// Send a pre-signed HTTP request (multipart operations) + fn send_raw( + &self, + method: http::Method, + url: String, + headers: HeaderMap, + body: Bytes, + ) -> impl Future> + MaybeSend; +} +``` + +## Three Responsibilities + +### `create_store()` + +Returns an `Arc` used only for LIST operations. The runtime may need to inject a custom HTTP connector: + +```rust +fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + // Use the shared helper, optionally injecting a custom connector + build_object_store(config, |builder| { + match builder { + StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(MyConnector)), + other => other, + } + }) +} +``` + +### `create_signer()` + +Returns an `Arc` for generating presigned URLs. Signing is pure computation — no HTTP connector needed: + +```rust +fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { + build_signer(config) +} +``` + +### `send_raw()` + +Executes a pre-signed HTTP request for multipart operations. Use your platform's HTTP client: + +```rust +async fn send_raw( + &self, + method: http::Method, + url: String, + headers: HeaderMap, + body: Bytes, +) -> Result { + let response = self.http_client + .request(method, &url) + .headers(headers) + .body(body) + .send() + .await + .map_err(|e| ProxyError::BackendError(e.to_string()))?; + + Ok(RawResponse { + status: response.status(), + headers: response.headers().clone(), + body: response.bytes().await + .map_err(|e| ProxyError::BackendError(e.to_string()))?, + }) +} +``` + +## Helper Functions + +The `backend` module provides shared helpers: + +- **`build_object_store(config, connector_fn)`** — Dispatches on `backend_type` ("s3", "az", "gcs"), iterates `backend_options` with `with_config()`, and applies the connector function +- **`build_signer(config)`** — Returns the appropriate signer: `object_store`'s built-in signer for authenticated backends, or `UnsignedUrlSigner` for anonymous backends + +These handle the multi-provider dispatch logic so your backend implementation only needs to provide the HTTP transport layer. + +## Wiring Into the Handler + +```rust +let backend = MyBackend::new(http_client); +let resolver = DefaultResolver::new(config_provider, token_key, domain); +let handler = ProxyHandler::new(backend, resolver); + +// In your request handler, handle all three action types: +match handler.resolve_request(method, path, query, &headers).await { + HandlerAction::Forward(fwd) => { + // Execute presigned URL with your HTTP client + // Stream request body (PUT) or response body (GET) + } + HandlerAction::Response(res) => { + // Return the complete response (LIST, errors) + } + HandlerAction::NeedsBody(pending) => { + // Collect request body, then: + let result = handler.handle_with_body(pending, body).await; + // Return the result + } +} +``` diff --git a/docs/extending/custom-provider.md b/docs/extending/custom-provider.md new file mode 100644 index 0000000..edaeaf2 --- /dev/null +++ b/docs/extending/custom-provider.md @@ -0,0 +1,104 @@ +# Custom Config Provider + +The `ConfigProvider` trait defines how the proxy loads buckets, roles, and credentials. Implement it to plug in your own configuration backend. + +## The Trait + +```rust +use source_coop_core::config::ConfigProvider; +use source_coop_core::error::ProxyError; +use source_coop_core::types::*; + +pub trait ConfigProvider: Clone + MaybeSend + MaybeSync + 'static { + async fn list_buckets(&self) -> Result, ProxyError>; + async fn get_bucket(&self, name: &str) -> Result, ProxyError>; + async fn get_role(&self, role_id: &str) -> Result, ProxyError>; + async fn get_credential(&self, access_key_id: &str) + -> Result, ProxyError>; +} +``` + +## Example: Redis Provider + +```rust +use source_coop_core::config::ConfigProvider; +use source_coop_core::error::ProxyError; +use source_coop_core::types::*; + +#[derive(Clone)] +struct RedisProvider { + client: redis::Client, +} + +impl ConfigProvider for RedisProvider { + async fn list_buckets(&self) -> Result, ProxyError> { + let mut conn = self.client.get_async_connection().await + .map_err(|e| ProxyError::Internal(e.to_string()))?; + + let keys: Vec = redis::cmd("KEYS") + .arg("bucket:*") + .query_async(&mut conn) + .await + .map_err(|e| ProxyError::Internal(e.to_string()))?; + + let mut buckets = Vec::new(); + for key in keys { + let json: String = redis::cmd("GET") + .arg(&key) + .query_async(&mut conn) + .await + .map_err(|e| ProxyError::Internal(e.to_string()))?; + let bucket: BucketConfig = serde_json::from_str(&json) + .map_err(|e| ProxyError::ConfigError(e.to_string()))?; + buckets.push(bucket); + } + Ok(buckets) + } + + async fn get_bucket(&self, name: &str) -> Result, ProxyError> { + // Similar Redis GET with key "bucket:{name}" + todo!() + } + + async fn get_role(&self, role_id: &str) -> Result, ProxyError> { + todo!() + } + + async fn get_credential(&self, access_key_id: &str) + -> Result, ProxyError> { + todo!() + } +} +``` + +## Using with DefaultResolver + +Wrap your provider in `DefaultResolver` to get standard S3 proxy behavior (path/virtual-host parsing, SigV4 auth, scope-based authorization): + +```rust +use source_coop_core::resolver::DefaultResolver; +use source_coop_core::config::cached::CachedProvider; +use std::time::Duration; + +// Optional: wrap with caching +let cached = CachedProvider::new(redis_provider, Duration::from_secs(60)); + +// Create resolver with optional token key and domain +let resolver = DefaultResolver::new(cached, token_key, virtual_host_domain); + +// Wire into the proxy handler +let handler = ProxyHandler::new(backend, resolver); +``` + +## Using with CachedProvider + +For network-backed providers, wrap with `CachedProvider` to reduce latency: + +```rust +use source_coop_core::config::cached::CachedProvider; +use std::time::Duration; + +let provider = CachedProvider::new(redis_provider, Duration::from_secs(120)); +``` + +See [Caching](/configuration/providers/cached) for cache behavior details. diff --git a/docs/extending/custom-resolver.md b/docs/extending/custom-resolver.md new file mode 100644 index 0000000..ec78735 --- /dev/null +++ b/docs/extending/custom-resolver.md @@ -0,0 +1,128 @@ +# Custom Request Resolver + +The `RequestResolver` trait controls how incoming requests are parsed, authenticated, and authorized. Implement it for full control over the request handling pipeline. + +## The Trait + +```rust +use source_coop_core::resolver::{RequestResolver, ResolvedAction}; +use source_coop_core::error::ProxyError; +use http::{Method, HeaderMap}; + +pub trait RequestResolver: Clone + MaybeSend + MaybeSync + 'static { + fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> impl Future> + MaybeSend; +} +``` + +## ResolvedAction + +The resolver returns one of two actions: + +```rust +pub enum ResolvedAction { + /// Forward to a backend (standard proxy behavior) + Proxy { + operation: S3Operation, + bucket_config: BucketConfig, + list_rewrite: Option, + }, + /// Return a synthetic response (e.g., virtual listing, redirect) + Response { + status: StatusCode, + headers: HeaderMap, + body: ProxyResponseBody, + }, +} +``` + +## Example: Custom Namespace + +A resolver that maps `/{account}/{repo}/{key}` to backend buckets: + +```rust +use source_coop_core::resolver::{RequestResolver, ResolvedAction}; +use source_coop_core::s3::request::build_s3_operation; +use source_coop_core::error::ProxyError; + +#[derive(Clone)] +struct MyResolver { + api_client: ApiClient, +} + +impl RequestResolver for MyResolver { + async fn resolve( + &self, + method: &Method, + path: &str, + query: Option<&str>, + headers: &HeaderMap, + ) -> Result { + // Parse custom URL structure + let parts: Vec<&str> = path.trim_start_matches('/').splitn(3, '/').collect(); + let (account, repo, key) = match parts.as_slice() { + [a, r, k] => (*a, *r, *k), + [a, r] => (*a, *r, ""), + _ => return Err(ProxyError::BucketNotFound), + }; + + // Look up the backend config from an external API + let bucket_config = self.api_client + .get_backend(account, repo) + .await + .map_err(|_| ProxyError::BucketNotFound)?; + + // Authenticate via external service + self.api_client + .check_permissions(account, repo, headers) + .await + .map_err(|_| ProxyError::AccessDenied)?; + + // Build the S3 operation from method + key + let operation = build_s3_operation(method, &bucket_config.name, key, query)?; + + Ok(ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite: None, + }) + } +} +``` + +## Wiring Into the Handler + +```rust +let resolver = MyResolver::new(api_client); +let handler = ProxyHandler::new(backend, resolver); + +// In your request handler: +let action = handler.resolve_request(method, path, query, &headers).await; +match action { + HandlerAction::Forward(fwd) => { /* execute presigned URL */ } + HandlerAction::Response(res) => { /* return response */ } + HandlerAction::NeedsBody(pending) => { /* collect body, call handle_with_body */ } +} +``` + +## ListRewrite + +The `ListRewrite` option in `ResolvedAction::Proxy` allows you to transform `` and `` values in LIST response XML: + +```rust +ResolvedAction::Proxy { + operation, + bucket_config, + list_rewrite: Some(ListRewrite { + strip_prefix: "internal/mirror/".to_string(), + add_prefix: "public/".to_string(), + }), +} +``` + +This is useful when the backend key structure differs from what clients expect. diff --git a/docs/extending/index.md b/docs/extending/index.md new file mode 100644 index 0000000..2c64b09 --- /dev/null +++ b/docs/extending/index.md @@ -0,0 +1,17 @@ +# Extending the Proxy + +The proxy is designed for customization through three trait boundaries. Each controls a different aspect of the proxy's behavior. + +| Trait | Controls | Default Implementation | +|-------|----------|----------------------| +| [RequestResolver](./custom-resolver) | How requests are parsed, authenticated, and authorized | `DefaultResolver` (standard S3 proxy behavior) | +| [ConfigProvider](./custom-provider) | Where configuration comes from | Static file, HTTP, DynamoDB, Postgres | +| [ProxyBackend](./custom-backend) | How the runtime interacts with backends | `ServerBackend`, `WorkerBackend` | + +## When to Customize What + +**Custom Resolver** — Your URL namespace doesn't map to `/{bucket}/{key}`, or you need external authorization (e.g., an API call), or you want different authentication logic. + +**Custom Config Provider** — You want to store config in a backend not already supported (e.g., etcd, Redis, Consul), or you need to derive config from another source. + +**Custom Backend** — You're deploying to a runtime that's neither a standard server nor Cloudflare Workers (e.g., AWS Lambda, Deno Deploy), or you need a different HTTP client. diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md new file mode 100644 index 0000000..0d5ea6a --- /dev/null +++ b/docs/getting-started/index.md @@ -0,0 +1,63 @@ +# Quick Start + +This guide is for administrators setting up and running the Source Data Proxy. If you're a user looking to access data through an existing proxy, see the [User Guide](/guide/). + +The Source Data Proxy is a multi-runtime S3 gateway that proxies requests to backend object stores. This guide gets you running locally in minutes. + +## Prerequisites + +- [Rust](https://www.rust-lang.org/tools/install) (latest stable) +- [Docker](https://docs.docker.com/get-docker/) (for local development with MinIO) + +## Start the Backend + +Use Docker Compose to start MinIO as a local object store: + +```bash +docker compose up +``` + +This starts: +- MinIO API on port `9000` +- MinIO Console on port `9001` (user: `minioadmin`, password: `minioadmin`) +- A seed job that creates example buckets with test data + +## Run the Proxy + +Choose either the native server runtime or Cloudflare Workers: + +::: code-group + +```bash [Server Runtime] +cargo run -p source-coop-server -- \ + --config config.local.toml \ + --listen 0.0.0.0:8080 +``` + +```bash [Cloudflare Workers] +cd crates/runtimes/cf-workers && npx wrangler dev +``` + +::: + +The server runtime listens on port `8080`. The Workers runtime listens on port `8787`. + +## Make Your First Request + +```bash +# Anonymous read from a public bucket +curl http://localhost:8080/public-data/hello.txt + +# Signed upload with the local dev credential +AWS_ACCESS_KEY_ID=AKLOCAL0000000000001 \ +AWS_SECRET_ACCESS_KEY="localdev/secret/key/00000000000000000000" \ +aws s3 cp ./myfile.txt s3://private-uploads/myfile.txt \ + --endpoint-url http://localhost:8080 +``` + +## Next Steps + +- [Local Development](./local-development) — Detailed dev environment setup +- [Configuration](/configuration/) — Configuring buckets, roles, and credentials +- [Authentication](/auth/) — Setting up auth flows +- [Deployment](/deployment/) — Deploying to production diff --git a/docs/getting-started/local-development.md b/docs/getting-started/local-development.md new file mode 100644 index 0000000..22e242b --- /dev/null +++ b/docs/getting-started/local-development.md @@ -0,0 +1,105 @@ +# Local Development + +This guide walks through setting up a full local development environment with MinIO as the backing object store. + +## Docker Compose + +The project includes a `docker-compose.yml` that starts MinIO and seeds it with example data: + +```bash +docker compose up +``` + +This starts: +- **MinIO API** at `http://localhost:9000` +- **MinIO Console** at `http://localhost:9001` (credentials: `minioadmin` / `minioadmin`) +- A seed job that creates `public-data` and `private-uploads` buckets with sample files + +## Configuration Files + +The two runtimes use different config formats: + +### Server Runtime — `config.local.toml` + +The server runtime reads a TOML config file. The local development config points buckets at `http://localhost:9000` (MinIO): + +```bash +cargo run -p source-coop-server -- \ + --config config.local.toml \ + --listen 0.0.0.0:8080 +``` + +### Workers Runtime — `wrangler.toml` + +The CF Workers runtime reads `PROXY_CONFIG` from the Wrangler configuration. It can be a JSON string or a JS object: + +```bash +cd crates/runtimes/cf-workers && npx wrangler dev +``` + +The Workers dev server runs on port `8787` by default. + +## Building + +```bash +# Check/build default workspace members (excludes cf-workers) +cargo check +cargo build + +# CF Workers must target wasm32 +cargo check -p source-coop-cf-workers --target wasm32-unknown-unknown + +# Run tests +cargo test +``` + +## Makefile + +The project includes a Makefile with common tasks: + +```bash +make check # cargo check +make check-wasm # cargo check for CF Workers (wasm32 target) +make test # cargo test +make fmt # check formatting +make clippy # run linter +make run-server # run the server runtime +make run-workers # run the workers runtime (wrangler dev) +make ci-fast # fmt + clippy + check-wasm +make ci # ci-fast + test +``` + +## Environment Variables + +For local development, these are optional but useful: + +| Variable | Purpose | Example | +|----------|---------|---------| +| `SESSION_TOKEN_KEY` | AES-256-GCM key for sealed tokens | `openssl rand -base64 32` | +| `OIDC_PROVIDER_KEY` | RSA private key for OIDC backend auth | PEM file contents | +| `OIDC_PROVIDER_ISSUER` | Public URL for OIDC discovery | `http://localhost:8080` | +| `RUST_LOG` | Logging level | `source_coop=debug` | + +## Verifying the Setup + +Once the proxy is running, test both anonymous and authenticated access: + +```bash +# Anonymous read (should return file contents) +curl http://localhost:8080/public-data/hello.txt + +# Authenticated upload +AWS_ACCESS_KEY_ID=AKLOCAL0000000000001 \ +AWS_SECRET_ACCESS_KEY="localdev/secret/key/00000000000000000000" \ +aws s3 cp ./test.txt s3://private-uploads/test.txt \ + --endpoint-url http://localhost:8080 + +# List bucket contents +AWS_ACCESS_KEY_ID=AKLOCAL0000000000001 \ +AWS_SECRET_ACCESS_KEY="localdev/secret/key/00000000000000000000" \ +aws s3 ls s3://private-uploads/ \ + --endpoint-url http://localhost:8080 + +# Browse MinIO directly +open http://localhost:9001 +``` diff --git a/docs/guide/authentication.md b/docs/guide/authentication.md new file mode 100644 index 0000000..82ab47c --- /dev/null +++ b/docs/guide/authentication.md @@ -0,0 +1,135 @@ +# Authentication + +The proxy supports three ways to authenticate, depending on your use case. + +## Anonymous Access + +Public buckets serve read requests without credentials: + +```bash +curl https://data.source.coop/public-data/hello.txt +``` + +Anonymous access only allows `GetObject`, `HeadObject`, and `ListBucket` operations. Write operations always require authentication. + +## Long-Lived Access Keys + +If your administrator has issued you a static access key pair, use them like standard AWS credentials: + +```bash +AWS_ACCESS_KEY_ID=AKPROXY00000EXAMPLE \ +AWS_SECRET_ACCESS_KEY="proxy/secret/key/EXAMPLE000000000000" \ +aws s3 cp s3://my-bucket/path/to/file.txt ./file.txt \ + --endpoint-url https://data.source.coop +``` + +These work with any S3-compatible client. The proxy verifies requests using standard AWS SigV4 signing, so no special client configuration is needed beyond setting the endpoint URL. + +## OIDC / STS Temporary Credentials + +This is the recommended authentication method. You exchange a JWT from your organization's identity provider for scoped, time-limited credentials — the same flow as AWS `AssumeRoleWithWebIdentity`. + +There are two ways to do this: the CLI (for interactive use) and direct STS calls (for CI/CD and scripts). + +### CLI Authentication + +The `source-coop` CLI handles the OIDC flow for you. It opens your browser, authenticates with your identity provider, and obtains temporary credentials. + +**Install the CLI:** + +```bash +cargo install --path crates/cli +``` + +**Log in:** + +```bash +source-coop login +``` + +This opens your browser to authenticate. Once complete, credentials are cached locally. + +### AWS Profile Integration + +Set up an AWS profile to use the proxy seamlessly with standard AWS tools: + +```ini +[profile source-coop] +credential_process = source-coop credential-process +endpoint_url = https://data.source.coop +``` + +For local development, override the proxy URL: + +```ini +[profile sc-local] +credential_process = source-coop credential-process --proxy-url http://localhost:8787 +endpoint_url = http://localhost:8787 +``` + +Then use AWS tools normally — credentials are obtained and refreshed automatically: + +```bash +aws s3 ls s3://my-bucket/ --profile source-coop +aws s3 cp ./data.csv s3://my-bucket/uploads/data.csv --profile source-coop +``` + +### Multiple Roles + +If your administrator has set up multiple roles with different access scopes, you can create a profile for each: + +```bash +source-coop login --role-arn reader-role +source-coop login --role-arn admin-role +``` + +```ini +[profile sc-reader] +credential_process = source-coop credential-process --role-arn reader-role +endpoint_url = https://data.source.coop + +[profile sc-admin] +credential_process = source-coop credential-process --role-arn admin-role +endpoint_url = https://data.source.coop +``` + +### CLI Options + +| Flag | Env Var | Default | Description | +|------|---------|---------|-------------| +| `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | +| `--client-id` | `SOURCE_OIDC_CLIENT_ID` | (built-in) | OAuth2 client ID | +| `--proxy-url` | `SOURCE_PROXY_URL` | `https://data.source.coop` | Proxy URL for STS | +| `--role-arn` | `SOURCE_ROLE_ARN` | `source-coop-user` | Role ARN to assume | +| `--format` | | `credential-process` | Output: `credential-process` or `env` | +| `--duration` | | (role default) | Session duration in seconds | +| `--scope` | | `openid` | OAuth2 scopes | +| `--port` | | `0` (random) | Local callback server port | + +### Direct STS Exchange + +For CI/CD pipelines and scripts, the CLI can output credentials as environment variables using `--format env`: + +```bash +eval $(source-coop login --format env) + +# Credentials are now exported — use any S3 client +aws s3 cp ./data.csv s3://deploy-bundles/data.csv \ + --endpoint-url https://data.source.coop +``` + +You can also call the STS endpoint directly with a JWT: + +```bash +CREDS=$(aws sts assume-role-with-web-identity \ + --role-arn github-actions-deployer \ + --web-identity-token "$JWT_TOKEN" \ + --endpoint-url https://data.source.coop \ + --output json) + +export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') +export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') +export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') +``` + +The STS endpoint accepts a JWT from any OIDC provider that your administrator has configured as trusted. See the [Administration guide](/auth/proxy-auth) for details on setting up identity providers and trust policies. diff --git a/docs/guide/client-usage.md b/docs/guide/client-usage.md new file mode 100644 index 0000000..638d834 --- /dev/null +++ b/docs/guide/client-usage.md @@ -0,0 +1,109 @@ +# Client Usage + +The proxy exposes a standard S3-compatible API. Any S3 client works — just set the endpoint URL to point at the proxy. + +## aws-cli + +```bash +# Download a file +aws s3 cp s3://my-bucket/path/to/file.txt ./file.txt \ + --endpoint-url https://data.source.coop + +# Upload a file +aws s3 cp ./local-file.txt s3://my-bucket/uploads/file.txt \ + --endpoint-url https://data.source.coop + +# List bucket contents +aws s3 ls s3://my-bucket/prefix/ \ + --endpoint-url https://data.source.coop +``` + +### Using AWS Profiles + +Add a profile to `~/.aws/config` to avoid specifying the endpoint every time: + +```ini +[profile source-coop] +credential_process = source-coop credential-process +endpoint_url = https://data.source.coop +``` + +Then use it: + +```bash +aws s3 ls s3://my-bucket/ --profile source-coop +aws s3 cp s3://my-bucket/data.csv ./data.csv --profile source-coop +``` + +See [Authentication](./authentication) for setting up credentials and profiles. + +## boto3 (Python) + +```python +import boto3 + +s3 = boto3.client( + "s3", + endpoint_url="https://data.source.coop", + aws_access_key_id="AKPROXY00000EXAMPLE", + aws_secret_access_key="proxy/secret/key/EXAMPLE", +) + +# Download +s3.download_file("my-bucket", "path/to/file.txt", "./file.txt") + +# Upload +s3.upload_file("./local-file.txt", "my-bucket", "uploads/file.txt") + +# List +response = s3.list_objects_v2(Bucket="my-bucket", Prefix="prefix/") +for obj in response.get("Contents", []): + print(obj["Key"]) +``` + +### Using a Profile with boto3 + +If you have an AWS profile configured with `credential_process`: + +```python +import boto3 + +session = boto3.Session(profile_name="source-coop") +s3 = session.client("s3") + +response = s3.list_objects_v2(Bucket="my-bucket") +``` + +## curl + +For anonymous buckets, you can use curl directly: + +```bash +# Download +curl https://data.source.coop/public-data/hello.txt + +# HEAD request (metadata only) +curl -I https://data.source.coop/public-data/hello.txt +``` + +For authenticated requests, use aws-cli or an SDK that handles SigV4 signing. + +## Request Styles + +The proxy supports two S3 URL styles: + +### Path Style (default) + +``` +https://data.source.coop/bucket-name/key/path +``` + +This is the default and works without additional configuration. + +### Virtual-Hosted Style + +``` +https://bucket-name.s3.example.com/key/path +``` + +Virtual-hosted style requires that the proxy administrator has configured the `--domain` flag. The proxy extracts the bucket name from the `Host` header. diff --git a/docs/guide/endpoints.md b/docs/guide/endpoints.md new file mode 100644 index 0000000..1d38c1a --- /dev/null +++ b/docs/guide/endpoints.md @@ -0,0 +1,102 @@ +# Endpoints + +Source Cooperative runs two types of proxy deployments. Choosing the right endpoint can significantly improve throughput and reduce costs. + +## Global Endpoint (Cloudflare Workers) + +``` +https://data.source.coop +``` + +The primary endpoint runs on Cloudflare Workers at the edge. This is the default for most use cases: + +- **Global availability** — Requests are handled by the nearest Cloudflare edge location +- **Backbone routing** — Traffic between Cloudflare and AWS travels over Cloudflare's private backbone network rather than the public internet, improving throughput and reliability +- **Best for** — Workstations, laptops, CI/CD outside AWS, or any client not running inside an AWS region + +```bash +aws s3 cp s3://my-bucket/data.parquet ./data.parquet \ + --endpoint-url https://data.source.coop +``` + +## Regional Endpoints (AWS Servers) + +``` +https://{region}.data.source.coop +``` + +For workloads running inside AWS, zone-specific server deployments are available. These run as native Tokio/Hyper servers within the same AWS region as the backend storage: + +| Endpoint | Region | +|----------|--------| +| `us-west-2.data.source.coop` | US West (Oregon) | +| `us-east-1.data.source.coop` | US East (N. Virginia) | + +Regional endpoints provide two major advantages: + +- **Higher throughput** — Traffic stays within the AWS network, avoiding internet bottlenecks. This is especially impactful for large file transfers and batch processing workloads +- **No egress fees** — Data transferred between S3 and an EC2 instance (or other AWS service) in the same region incurs no AWS data transfer charges. Using the global endpoint from within AWS would route traffic out through Cloudflare and back, incurring egress fees on both legs + +### When to Use Regional Endpoints + +Use a regional endpoint when your client is running inside the same AWS region as the data: + +- **EC2 instances** processing datasets stored in the same region +- **SageMaker notebooks** or training jobs accessing training data +- **Lambda functions** reading/writing data in batch pipelines +- **ECS/EKS workloads** performing ETL or analytics +- **AWS Batch** jobs processing large datasets + +```bash +# From an EC2 instance in us-west-2 +aws s3 cp s3://my-bucket/large-dataset.parquet ./data.parquet \ + --endpoint-url https://us-west-2.data.source.coop +``` + +### AWS Profile Configuration + +Set up profiles for both global and regional access: + +```ini +# For general use (laptop, CI/CD outside AWS) +[profile source-coop] +credential_process = source-coop credential-process +endpoint_url = https://data.source.coop + +# For workloads in us-west-2 +[profile source-coop-usw2] +credential_process = source-coop credential-process +endpoint_url = https://us-west-2.data.source.coop +``` + +```bash +# From your laptop +aws s3 ls s3://my-bucket/ --profile source-coop + +# From an EC2 instance in us-west-2 +aws s3 ls s3://my-bucket/ --profile source-coop-usw2 +``` + +## Choosing an Endpoint + +```mermaid +flowchart TD + Start["Where is your client running?"] + Start -->|Inside AWS| Region["Same region as the data?"] + Start -->|Outside AWS| Global["Use data.source.coop"] + + Region -->|Yes| Regional["Use {region}.data.source.coop"] + Region -->|No / Unsure| Global +``` + +| Scenario | Recommended Endpoint | Why | +|----------|---------------------|-----| +| Laptop or workstation | `data.source.coop` | Cloudflare backbone optimizes global routing | +| GitHub Actions / CI | `data.source.coop` | CI runners are typically outside AWS | +| EC2 in us-west-2, data in us-west-2 | `us-west-2.data.source.coop` | Same-region: max throughput, zero egress | +| EC2 in us-east-1, data in us-west-2 | `data.source.coop` | Cross-region: Cloudflare backbone is faster than cross-region AWS traffic | +| SageMaker in us-west-2 | `us-west-2.data.source.coop` | Same-region: zero egress for training data | + +::: tip +All endpoints support the same authentication methods and S3 operations. Your credentials work across any endpoint — only the `endpoint_url` changes. +::: diff --git a/docs/guide/index.md b/docs/guide/index.md new file mode 100644 index 0000000..3766cc7 --- /dev/null +++ b/docs/guide/index.md @@ -0,0 +1,24 @@ +# User Guide + +The Source Data Proxy provides S3-compatible access to data stored across multiple cloud backends. You interact with it using standard S3 tools — `aws-cli`, `boto3`, or any S3-compatible SDK — just point the endpoint URL at the proxy. + +## Getting Started + +1. **[Endpoints](./endpoints)** — Which endpoint to use (global vs. regional) +2. **[Authentication](./authentication)** — How to authenticate and obtain credentials +3. **[Client Usage](./client-usage)** — Using aws-cli, boto3, curl, and other S3 clients + +## Quick Example + +```bash +# Anonymous access to a public bucket +curl https://data.source.coop/public-data/hello.txt + +# Authenticated access with the CLI +source-coop login +aws s3 ls s3://my-bucket/ --profile source-coop +``` + +## How It Works + +The proxy sits between your S3 client and the backend object stores. You send standard S3 requests to the proxy, and it handles authentication, authorization, and forwarding to the correct backend. From your perspective, it behaves like any other S3-compatible service. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..ed07d26 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,79 @@ +--- +layout: home + +hero: + name: Source Data Proxy + text: Multi-runtime S3 gateway proxy + tagline: A Radiant Earth project. Stream S3-compatible requests to backend object stores with authentication, authorization, and zero-copy passthrough. + actions: + - theme: brand + text: User Guide + link: /guide/ + - theme: alt + text: Administration + link: /getting-started/ + - theme: alt + text: View on GitHub + link: https://github.com/source-cooperative/data.source.coop + +features: + - title: Unified Interface + details: One stable URL per dataset, regardless of which object storage provider hosts the bytes. Backend migrations are invisible to data consumers. + - title: Native S3 Compatibility + details: Works with aws-cli, boto3, DuckDB, the object_store crate, GDAL, and any S3-compatible client. No custom SDK — just set the endpoint URL. + - title: Metered Access + details: Enforce per-identity rate limits so open data stays free for humans while protecting infrastructure from runaway machine access and egress costs. + - title: Flexible Auth + details: OIDC token exchange for both frontend (user/machine identity) and backend (cloud storage credentials). No long-lived keys anywhere in the chain. + - title: Multi-Runtime + details: Same core logic deploys as a native Tokio/Hyper server in containers or as a Cloudflare Worker at the edge. + - title: Zero-Copy Streaming + details: Presigned URLs enable direct streaming between clients and backends. No buffering, no double-handling of request or response bodies. +--- + +## Why a Proxy? + +Source Cooperative hosts open data from researchers and organizations around the world. That data lives on object storage — but object storage alone doesn't solve the problems that come with making data truly accessible. + +### One URL, any backend + +A dataset might start on AWS S3, move to Cloudflare R2 to reduce egress costs, or get mirrored across providers for redundancy. The proxy gives every data product a stable URL (`data.source.coop/{account}/{dataset}/...`) regardless of where the bytes actually live. Backend migrations are invisible to consumers — no broken links, no client reconfiguration. + +### Native S3 compatibility + +Rather than inventing a new API, the proxy speaks the S3 protocol. This means the entire ecosystem of existing tools — `aws-cli`, `boto3`, DuckDB, the Rust `object_store` crate, GDAL, and hundreds of others — works out of the box. Users don't install a custom client or learn a new SDK. They just set an endpoint URL. + +### Metered access + +Open data should be free and open to humans, but without guardrails a single runaway script can rack up thousands of dollars in egress charges. The proxy enables metered access — enforcing limits on how much data a given identity can consume in a window of time. Public datasets stay freely accessible while the infrastructure stays sustainable. + +### Flexible authentication + +The proxy supports two layers of OIDC-based auth that eliminate long-lived credentials: + +- **Frontend**: Third-party identity providers (GitHub Actions, Auth0, Keycloak) can exchange OIDC tokens for scoped, time-limited proxy credentials — enabling machine-to-machine workflows like ETL pipelines and CI/CD without sharing static keys. +- **Backend**: The proxy acts as its own OIDC identity provider to authenticate with cloud storage backends, replacing long-lived access keys with short-lived credentials obtained via token exchange. + +### Run anywhere + +The same core logic compiles to a native Tokio/Hyper server for container deployments and to WebAssembly for Cloudflare Workers at the edge. Choose the runtime that fits your infrastructure — or run both. + +## How It Works + +```mermaid +flowchart LR + Clients["S3 Clients
(aws-cli, boto3, SDKs)"] + + subgraph Proxy["source-coop-proxy"] + Auth["Auth
(STS, OIDC, SigV4)"] + Core["Core
(Proxy Handler)"] + Config["Config
(Static, HTTP, DynamoDB, Postgres)"] + end + + Backend["Backend Stores
(AWS S3, MinIO, R2, Azure, GCS)"] + + Clients <--> Proxy + Proxy <--> Backend +``` + +The proxy sits between S3-compatible clients and backend object stores. It authenticates incoming requests, authorizes them against configured scopes, and forwards them to the appropriate backend using presigned URLs for zero-copy streaming. diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000..7f2004e --- /dev/null +++ b/docs/package.json @@ -0,0 +1,20 @@ +{ + "name": "source-data-proxy-docs", + "private": true, + "type": "module", + "scripts": { + "docs:dev": "vitepress dev", + "docs:build": "vitepress build", + "docs:preview": "vitepress preview" + }, + "devDependencies": { + "@braintree/sanitize-url": "^7.1.2", + "cytoscape": "^3.33.1", + "cytoscape-cose-bilkent": "^4.1.0", + "dayjs": "^1.11.19", + "debug": "^4.4.3", + "mermaid": "^11.4.1", + "vitepress": "^1.6.3", + "vitepress-plugin-mermaid": "^2.0.17" + } +} diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml new file mode 100644 index 0000000..5376ad6 --- /dev/null +++ b/docs/pnpm-lock.yaml @@ -0,0 +1,2620 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@braintree/sanitize-url': + specifier: ^7.1.2 + version: 7.1.2 + cytoscape: + specifier: ^3.33.1 + version: 3.33.1 + cytoscape-cose-bilkent: + specifier: ^4.1.0 + version: 4.1.0(cytoscape@3.33.1) + dayjs: + specifier: ^1.11.19 + version: 1.11.19 + debug: + specifier: ^4.4.3 + version: 4.4.3 + mermaid: + specifier: ^11.4.1 + version: 11.12.3 + vitepress: + specifier: ^1.6.3 + version: 1.6.4(@algolia/client-search@5.49.1)(postcss@8.5.6)(search-insights@2.17.3) + vitepress-plugin-mermaid: + specifier: ^2.0.17 + version: 2.0.17(mermaid@11.12.3)(vitepress@1.6.4(@algolia/client-search@5.49.1)(postcss@8.5.6)(search-insights@2.17.3)) + +packages: + + '@algolia/abtesting@1.15.1': + resolution: {integrity: sha512-2yuIC48rUuHGhU1U5qJ9kJHaxYpJ0jpDHJVI5ekOxSMYXlH4+HP+pA31G820lsAznfmu2nzDV7n5RO44zIY1zw==} + engines: {node: '>= 14.0.0'} + + '@algolia/autocomplete-core@1.17.7': + resolution: {integrity: sha512-BjiPOW6ks90UKl7TwMv7oNQMnzU+t/wk9mgIDi6b1tXpUek7MW0lbNOUHpvam9pe3lVCf4xPFT+lK7s+e+fs7Q==} + + '@algolia/autocomplete-plugin-algolia-insights@1.17.7': + resolution: {integrity: sha512-Jca5Ude6yUOuyzjnz57og7Et3aXjbwCSDf/8onLHSQgw1qW3ALl9mrMWaXb5FmPVkV3EtkD2F/+NkT6VHyPu9A==} + peerDependencies: + search-insights: '>= 1 < 3' + + '@algolia/autocomplete-preset-algolia@1.17.7': + resolution: {integrity: sha512-ggOQ950+nwbWROq2MOCIL71RE0DdQZsceqrg32UqnhDz8FlO9rL8ONHNsI2R1MH0tkgVIDKI/D0sMiUchsFdWA==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + + '@algolia/autocomplete-shared@1.17.7': + resolution: {integrity: sha512-o/1Vurr42U/qskRSuhBH+VKxMvkkUVTLU6WZQr+L5lGZZLYWyhdzWjW0iGXY7EkwRTjBqvN2EsR81yCTGV/kmg==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + + '@algolia/client-abtesting@5.49.1': + resolution: {integrity: sha512-h6M7HzPin+45/l09q0r2dYmocSSt2MMGOOk5c4O5K/bBBlEwf1BKfN6z+iX4b8WXcQQhf7rgQwC52kBZJt/ZZw==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-analytics@5.49.1': + resolution: {integrity: sha512-048T9/Z8OeLmTk8h76QUqaNFp7Rq2VgS2Zm6Y2tNMYGQ1uNuzePY/udB5l5krlXll7ZGflyCjFvRiOtlPZpE9g==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-common@5.49.1': + resolution: {integrity: sha512-vp5/a9ikqvf3mn9QvHN8PRekn8hW34aV9eX+O0J5mKPZXeA6Pd5OQEh2ZWf7gJY6yyfTlLp5LMFzQUAU+Fpqpg==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-insights@5.49.1': + resolution: {integrity: sha512-B6N7PgkvYrul3bntTz/l6uXnhQ2bvP+M7NqTcayh681tSqPaA5cJCUBp/vrP7vpPRpej4Eeyx2qz5p0tE/2N2g==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-personalization@5.49.1': + resolution: {integrity: sha512-v+4DN+lkYfBd01Hbnb9ZrCHe7l+mvihyx218INRX/kaCXROIWUDIT1cs3urQxfE7kXBFnLsqYeOflQALv/gA5w==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-query-suggestions@5.49.1': + resolution: {integrity: sha512-Un11cab6ZCv0W+Jiak8UktGIqoa4+gSNgEZNfG8m8eTsXGqwIEr370H3Rqwj87zeNSlFpH2BslMXJ/cLNS1qtg==} + engines: {node: '>= 14.0.0'} + + '@algolia/client-search@5.49.1': + resolution: {integrity: sha512-Nt9hri7nbOo0RipAsGjIssHkpLMHHN/P7QqENywAq5TLsoYDzUyJGny8FEiD/9KJUxtGH8blGpMedilI6kK3rA==} + engines: {node: '>= 14.0.0'} + + '@algolia/ingestion@1.49.1': + resolution: {integrity: sha512-b5hUXwDqje0Y4CpU6VL481DXgPgxpTD5sYMnfQTHKgUispGnaCLCm2/T9WbJo1YNUbX3iHtYDArp804eD6CmRQ==} + engines: {node: '>= 14.0.0'} + + '@algolia/monitoring@1.49.1': + resolution: {integrity: sha512-bvrXwZ0WsL3rN6Q4m4QqxsXFCo6WAew7sAdrpMQMK4Efn4/W920r9ptOuckejOSSvyLr9pAWgC5rsHhR2FYuYw==} + engines: {node: '>= 14.0.0'} + + '@algolia/recommend@5.49.1': + resolution: {integrity: sha512-h2yz3AGeGkQwNgbLmoe3bxYs8fac4An1CprKTypYyTU/k3Q+9FbIvJ8aS1DoBKaTjSRZVoyQS7SZQio6GaHbZw==} + engines: {node: '>= 14.0.0'} + + '@algolia/requester-browser-xhr@5.49.1': + resolution: {integrity: sha512-2UPyRuUR/qpqSqH8mxFV5uBZWEpxhGPHLlx9Xf6OVxr79XO2ctzZQAhsmTZ6X22x+N8MBWpB9UEky7YU2HGFgA==} + engines: {node: '>= 14.0.0'} + + '@algolia/requester-fetch@5.49.1': + resolution: {integrity: sha512-N+xlE4lN+wpuT+4vhNEwPVlrfN+DWAZmSX9SYhbz986Oq8AMsqdntOqUyiOXVxYsQtfLwmiej24vbvJGYv1Qtw==} + engines: {node: '>= 14.0.0'} + + '@algolia/requester-node-http@5.49.1': + resolution: {integrity: sha512-zA5bkUOB5PPtTr182DJmajCiizHp0rCJQ0Chf96zNFvkdESKYlDeYA3tQ7r2oyHbu/8DiohAQ5PZ85edctzbXA==} + engines: {node: '>= 14.0.0'} + + '@antfu/install-pkg@1.1.0': + resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.0': + resolution: {integrity: sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/types@7.29.0': + resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} + engines: {node: '>=6.9.0'} + + '@braintree/sanitize-url@6.0.4': + resolution: {integrity: sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==} + + '@braintree/sanitize-url@7.1.2': + resolution: {integrity: sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==} + + '@chevrotain/cst-dts-gen@11.1.2': + resolution: {integrity: sha512-XTsjvDVB5nDZBQB8o0o/0ozNelQtn2KrUVteIHSlPd2VAV2utEb6JzyCJaJ8tGxACR4RiBNWy5uYUHX2eji88Q==} + + '@chevrotain/gast@11.1.2': + resolution: {integrity: sha512-Z9zfXR5jNZb1Hlsd/p+4XWeUFugrHirq36bKzPWDSIacV+GPSVXdk+ahVWZTwjhNwofAWg/sZg58fyucKSQx5g==} + + '@chevrotain/regexp-to-ast@11.1.2': + resolution: {integrity: sha512-nMU3Uj8naWer7xpZTYJdxbAs6RIv/dxYzkYU8GSwgUtcAAlzjcPfX1w+RKRcYG8POlzMeayOQ/znfwxEGo5ulw==} + + '@chevrotain/types@11.1.2': + resolution: {integrity: sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==} + + '@chevrotain/utils@11.1.2': + resolution: {integrity: sha512-4mudFAQ6H+MqBTfqLmU7G1ZwRzCLfJEooL/fsF6rCX5eePMbGhoy5n4g+G4vlh2muDcsCTJtL+uKbOzWxs5LHA==} + + '@docsearch/css@3.8.2': + resolution: {integrity: sha512-y05ayQFyUmCXze79+56v/4HpycYF3uFqB78pLPrSV5ZKAlDuIAAJNhaRi8tTdRNXh05yxX/TyNnzD6LwSM89vQ==} + + '@docsearch/js@3.8.2': + resolution: {integrity: sha512-Q5wY66qHn0SwA7Taa0aDbHiJvaFJLOJyHmooQ7y8hlwwQLQ/5WwCcoX0g7ii04Qi2DJlHsd0XXzJ8Ypw9+9YmQ==} + + '@docsearch/react@3.8.2': + resolution: {integrity: sha512-xCRrJQlTt8N9GU0DG4ptwHRkfnSnD/YpdeaXe02iKfqs97TkZJv60yE+1eq/tjPcVnTW8dP5qLP7itifFVV5eg==} + peerDependencies: + '@types/react': '>= 16.8.0 < 19.0.0' + react: '>= 16.8.0 < 19.0.0' + react-dom: '>= 16.8.0 < 19.0.0' + search-insights: '>= 1 < 3' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + react-dom: + optional: true + search-insights: + optional: true + + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@iconify-json/simple-icons@1.2.71': + resolution: {integrity: sha512-rNoDFbq1fAYiEexBvrw613/xiUOPEu5MKVV/X8lI64AgdTzLQUUemr9f9fplxUMPoxCBP2rWzlhOEeTHk/Sf0Q==} + + '@iconify/types@2.0.0': + resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} + + '@iconify/utils@3.1.0': + resolution: {integrity: sha512-Zlzem1ZXhI1iHeeERabLNzBHdOa4VhQbqAcOQaMKuTuyZCpwKbC2R4Dd0Zo3g9EAc+Y4fiarO8HIHRAth7+skw==} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@mermaid-js/mermaid-mindmap@9.3.0': + resolution: {integrity: sha512-IhtYSVBBRYviH1Ehu8gk69pMDF8DSRqXBRDMWrEfHoaMruHeaP2DXA3PBnuwsMaCdPQhlUUcy/7DBLAEIXvCAw==} + + '@mermaid-js/parser@1.0.0': + resolution: {integrity: sha512-vvK0Hi/VWndxoh03Mmz6wa1KDriSPjS2XMZL/1l19HFwygiObEEoEwSDxOqyLzzAI6J2PU3261JjTMTO7x+BPw==} + + '@rollup/rollup-android-arm-eabi@4.59.0': + resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.59.0': + resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.59.0': + resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.59.0': + resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.59.0': + resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.59.0': + resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': + resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.59.0': + resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.59.0': + resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.59.0': + resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.59.0': + resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.59.0': + resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.59.0': + resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.59.0': + resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.59.0': + resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.59.0': + resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.59.0': + resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.59.0': + resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.59.0': + resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.59.0': + resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.59.0': + resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.59.0': + resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.59.0': + resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.59.0': + resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.59.0': + resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} + cpu: [x64] + os: [win32] + + '@shikijs/core@2.5.0': + resolution: {integrity: sha512-uu/8RExTKtavlpH7XqnVYBrfBkUc20ngXiX9NSrBhOVZYv/7XQRKUyhtkeflY5QsxC0GbJThCerruZfsUaSldg==} + + '@shikijs/engine-javascript@2.5.0': + resolution: {integrity: sha512-VjnOpnQf8WuCEZtNUdjjwGUbtAVKuZkVQ/5cHy/tojVVRIRtlWMYVjyWhxOmIq05AlSOv72z7hRNRGVBgQOl0w==} + + '@shikijs/engine-oniguruma@2.5.0': + resolution: {integrity: sha512-pGd1wRATzbo/uatrCIILlAdFVKdxImWJGQ5rFiB5VZi2ve5xj3Ax9jny8QvkaV93btQEwR/rSz5ERFpC5mKNIw==} + + '@shikijs/langs@2.5.0': + resolution: {integrity: sha512-Qfrrt5OsNH5R+5tJ/3uYBBZv3SuGmnRPejV9IlIbFH3HTGLDlkqgHymAlzklVmKBjAaVmkPkyikAV/sQ1wSL+w==} + + '@shikijs/themes@2.5.0': + resolution: {integrity: sha512-wGrk+R8tJnO0VMzmUExHR+QdSaPUl/NKs+a4cQQRWyoc3YFbUzuLEi/KWK1hj+8BfHRKm2jNhhJck1dfstJpiw==} + + '@shikijs/transformers@2.5.0': + resolution: {integrity: sha512-SI494W5X60CaUwgi8u4q4m4s3YAFSxln3tzNjOSYqq54wlVgz0/NbbXEb3mdLbqMBztcmS7bVTaEd2w0qMmfeg==} + + '@shikijs/types@2.5.0': + resolution: {integrity: sha512-ygl5yhxki9ZLNuNpPitBWvcy9fsSKKaRuO4BAlMyagszQidxcpLAr0qiW/q43DtSIDxO6hEbtYLiFZNXO/hdGw==} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + + '@types/d3-array@3.2.2': + resolution: {integrity: sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==} + + '@types/d3-axis@3.0.6': + resolution: {integrity: sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==} + + '@types/d3-brush@3.0.6': + resolution: {integrity: sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==} + + '@types/d3-chord@3.0.6': + resolution: {integrity: sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-contour@3.0.6': + resolution: {integrity: sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==} + + '@types/d3-delaunay@6.0.4': + resolution: {integrity: sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==} + + '@types/d3-dispatch@3.0.7': + resolution: {integrity: sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-dsv@3.0.7': + resolution: {integrity: sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==} + + '@types/d3-ease@3.0.2': + resolution: {integrity: sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==} + + '@types/d3-fetch@3.0.7': + resolution: {integrity: sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==} + + '@types/d3-force@3.0.10': + resolution: {integrity: sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==} + + '@types/d3-format@3.0.4': + resolution: {integrity: sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==} + + '@types/d3-geo@3.1.0': + resolution: {integrity: sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==} + + '@types/d3-hierarchy@3.1.7': + resolution: {integrity: sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-path@3.1.1': + resolution: {integrity: sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==} + + '@types/d3-polygon@3.0.2': + resolution: {integrity: sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==} + + '@types/d3-quadtree@3.0.6': + resolution: {integrity: sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==} + + '@types/d3-random@3.0.3': + resolution: {integrity: sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==} + + '@types/d3-scale-chromatic@3.1.0': + resolution: {integrity: sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==} + + '@types/d3-scale@4.0.9': + resolution: {integrity: sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-shape@3.1.8': + resolution: {integrity: sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==} + + '@types/d3-time-format@4.0.3': + resolution: {integrity: sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==} + + '@types/d3-time@3.0.4': + resolution: {integrity: sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==} + + '@types/d3-timer@3.0.2': + resolution: {integrity: sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + + '@types/d3@7.4.3': + resolution: {integrity: sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/geojson@7946.0.16': + resolution: {integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==} + + '@types/hast@3.0.4': + resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + + '@types/linkify-it@5.0.0': + resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} + + '@types/markdown-it@14.1.2': + resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + + '@types/mdurl@2.0.0': + resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} + + '@types/trusted-types@2.0.7': + resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + + '@types/web-bluetooth@0.0.21': + resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + + '@ungap/structured-clone@1.3.0': + resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + + '@vitejs/plugin-vue@5.2.4': + resolution: {integrity: sha512-7Yx/SXSOcQq5HiiV3orevHUFn+pmMB4cgbEkDYgnkUWb0WfeQ/wa2yFv6D5ICiCQOVpjA7vYDXrC7AGO8yjDHA==} + engines: {node: ^18.0.0 || >=20.0.0} + peerDependencies: + vite: ^5.0.0 || ^6.0.0 + vue: ^3.2.25 + + '@vue/compiler-core@3.5.29': + resolution: {integrity: sha512-cuzPhD8fwRHk8IGfmYaR4eEe4cAyJEL66Ove/WZL7yWNL134nqLddSLwNRIsFlnnW1kK+p8Ck3viFnC0chXCXw==} + + '@vue/compiler-dom@3.5.29': + resolution: {integrity: sha512-n0G5o7R3uBVmVxjTIYcz7ovr8sy7QObFG8OQJ3xGCDNhbG60biP/P5KnyY8NLd81OuT1WJflG7N4KWYHaeeaIg==} + + '@vue/compiler-sfc@3.5.29': + resolution: {integrity: sha512-oJZhN5XJs35Gzr50E82jg2cYdZQ78wEwvRO6Y63TvLVTc+6xICzJHP1UIecdSPPYIbkautNBanDiWYa64QSFIA==} + + '@vue/compiler-ssr@3.5.29': + resolution: {integrity: sha512-Y/ARJZE6fpjzL5GH/phJmsFwx3g6t2KmHKHx5q+MLl2kencADKIrhH5MLF6HHpRMmlRAYBRSvv347Mepf1zVNw==} + + '@vue/devtools-api@7.7.9': + resolution: {integrity: sha512-kIE8wvwlcZ6TJTbNeU2HQNtaxLx3a84aotTITUuL/4bzfPxzajGBOoqjMhwZJ8L9qFYDU/lAYMEEm11dnZOD6g==} + + '@vue/devtools-kit@7.7.9': + resolution: {integrity: sha512-PyQ6odHSgiDVd4hnTP+aDk2X4gl2HmLDfiyEnn3/oV+ckFDuswRs4IbBT7vacMuGdwY/XemxBoh302ctbsptuA==} + + '@vue/devtools-shared@7.7.9': + resolution: {integrity: sha512-iWAb0v2WYf0QWmxCGy0seZNDPdO3Sp5+u78ORnyeonS6MT4PC7VPrryX2BpMJrwlDeaZ6BD4vP4XKjK0SZqaeA==} + + '@vue/reactivity@3.5.29': + resolution: {integrity: sha512-zcrANcrRdcLtmGZETBxWqIkoQei8HaFpZWx/GHKxx79JZsiZ8j1du0VUJtu4eJjgFvU/iKL5lRXFXksVmI+5DA==} + + '@vue/runtime-core@3.5.29': + resolution: {integrity: sha512-8DpW2QfdwIWOLqtsNcds4s+QgwSaHSJY/SUe04LptianUQ/0xi6KVsu/pYVh+HO3NTVvVJjIPL2t6GdeKbS4Lg==} + + '@vue/runtime-dom@3.5.29': + resolution: {integrity: sha512-AHvvJEtcY9tw/uk+s/YRLSlxxQnqnAkjqvK25ZiM4CllCZWzElRAoQnCM42m9AHRLNJ6oe2kC5DCgD4AUdlvXg==} + + '@vue/server-renderer@3.5.29': + resolution: {integrity: sha512-G/1k6WK5MusLlbxSE2YTcqAAezS+VuwHhOvLx2KnQU7G2zCH6KIb+5Wyt6UjMq7a3qPzNEjJXs1hvAxDclQH+g==} + peerDependencies: + vue: 3.5.29 + + '@vue/shared@3.5.29': + resolution: {integrity: sha512-w7SR0A5zyRByL9XUkCfdLs7t9XOHUyJ67qPGQjOou3p6GvBeBW+AVjUUmlxtZ4PIYaRvE+1LmK44O4uajlZwcg==} + + '@vueuse/core@12.8.2': + resolution: {integrity: sha512-HbvCmZdzAu3VGi/pWYm5Ut+Kd9mn1ZHnn4L5G8kOQTPs/IwIAmJoBrmYk2ckLArgMXZj0AW3n5CAejLUO+PhdQ==} + + '@vueuse/integrations@12.8.2': + resolution: {integrity: sha512-fbGYivgK5uBTRt7p5F3zy6VrETlV9RtZjBqd1/HxGdjdckBgBM4ugP8LHpjolqTj14TXTxSK1ZfgPbHYyGuH7g==} + peerDependencies: + async-validator: ^4 + axios: ^1 + change-case: ^5 + drauu: ^0.4 + focus-trap: ^7 + fuse.js: ^7 + idb-keyval: ^6 + jwt-decode: ^4 + nprogress: ^0.2 + qrcode: ^1.5 + sortablejs: ^1 + universal-cookie: ^7 + peerDependenciesMeta: + async-validator: + optional: true + axios: + optional: true + change-case: + optional: true + drauu: + optional: true + focus-trap: + optional: true + fuse.js: + optional: true + idb-keyval: + optional: true + jwt-decode: + optional: true + nprogress: + optional: true + qrcode: + optional: true + sortablejs: + optional: true + universal-cookie: + optional: true + + '@vueuse/metadata@12.8.2': + resolution: {integrity: sha512-rAyLGEuoBJ/Il5AmFHiziCPdQzRt88VxR+Y/A/QhJ1EWtWqPBBAxTAFaSkviwEuOEZNtW8pvkPgoCZQ+HxqW1A==} + + '@vueuse/shared@12.8.2': + resolution: {integrity: sha512-dznP38YzxZoNloI0qpEfpkms8knDtaoQ6Y/sfS0L7Yki4zh40LFHEhur0odJC6xTHG5dxWVPiUWBXn+wCG2s5w==} + + acorn@8.16.0: + resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} + engines: {node: '>=0.4.0'} + hasBin: true + + algoliasearch@5.49.1: + resolution: {integrity: sha512-X3Pp2aRQhg4xUC6PQtkubn5NpRKuUPQ9FPDQlx36SmpFwwH2N0/tw4c+NXV3nw3PsgeUs+BuWGP0gjz3TvENLQ==} + engines: {node: '>= 14.0.0'} + + birpc@2.9.0: + resolution: {integrity: sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw==} + + ccount@2.0.1: + resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + + character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + + chevrotain-allstar@0.3.1: + resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} + peerDependencies: + chevrotain: ^11.0.0 + + chevrotain@11.1.2: + resolution: {integrity: sha512-opLQzEVriiH1uUQ4Kctsd49bRoFDXGGSC4GUqj7pGyxM3RehRhvTlZJc1FL/Flew2p5uwxa1tUDWKzI4wNM8pg==} + + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + + commander@7.2.0: + resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} + engines: {node: '>= 10'} + + commander@8.3.0: + resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} + engines: {node: '>= 12'} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + copy-anything@4.0.5: + resolution: {integrity: sha512-7Vv6asjS4gMOuILabD3l739tsaxFQmC+a7pLZm02zyvs8p977bL3zEgq3yDk5rn9B0PbYgIv++jmHcuUab4RhA==} + engines: {node: '>=18'} + + cose-base@1.0.3: + resolution: {integrity: sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==} + + cose-base@2.2.0: + resolution: {integrity: sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + cytoscape-cose-bilkent@4.1.0: + resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} + peerDependencies: + cytoscape: ^3.2.0 + + cytoscape-fcose@2.2.0: + resolution: {integrity: sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==} + peerDependencies: + cytoscape: ^3.2.0 + + cytoscape@3.33.1: + resolution: {integrity: sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==} + engines: {node: '>=0.10'} + + d3-array@2.12.1: + resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==} + + d3-array@3.2.4: + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} + engines: {node: '>=12'} + + d3-axis@3.0.0: + resolution: {integrity: sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==} + engines: {node: '>=12'} + + d3-brush@3.0.0: + resolution: {integrity: sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==} + engines: {node: '>=12'} + + d3-chord@3.0.1: + resolution: {integrity: sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==} + engines: {node: '>=12'} + + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-contour@4.0.2: + resolution: {integrity: sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==} + engines: {node: '>=12'} + + d3-delaunay@6.0.4: + resolution: {integrity: sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-dsv@3.0.1: + resolution: {integrity: sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==} + engines: {node: '>=12'} + hasBin: true + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-fetch@3.0.1: + resolution: {integrity: sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==} + engines: {node: '>=12'} + + d3-force@3.0.0: + resolution: {integrity: sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==} + engines: {node: '>=12'} + + d3-format@3.1.2: + resolution: {integrity: sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==} + engines: {node: '>=12'} + + d3-geo@3.1.1: + resolution: {integrity: sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==} + engines: {node: '>=12'} + + d3-hierarchy@3.1.2: + resolution: {integrity: sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-path@1.0.9: + resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} + + d3-path@3.1.0: + resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} + engines: {node: '>=12'} + + d3-polygon@3.0.1: + resolution: {integrity: sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==} + engines: {node: '>=12'} + + d3-quadtree@3.0.1: + resolution: {integrity: sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==} + engines: {node: '>=12'} + + d3-random@3.0.1: + resolution: {integrity: sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==} + engines: {node: '>=12'} + + d3-sankey@0.12.3: + resolution: {integrity: sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==} + + d3-scale-chromatic@3.1.0: + resolution: {integrity: sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==} + engines: {node: '>=12'} + + d3-scale@4.0.2: + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-shape@1.3.7: + resolution: {integrity: sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==} + + d3-shape@3.2.0: + resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} + engines: {node: '>=12'} + + d3-time-format@4.1.0: + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} + engines: {node: '>=12'} + + d3-time@3.1.0: + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + + d3@7.9.0: + resolution: {integrity: sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==} + engines: {node: '>=12'} + + dagre-d3-es@7.0.13: + resolution: {integrity: sha512-efEhnxpSuwpYOKRm/L5KbqoZmNNukHa/Flty4Wp62JRvgH2ojwVgPgdYyr4twpieZnyRDdIH7PY2mopX26+j2Q==} + + dayjs@1.11.19: + resolution: {integrity: sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delaunator@5.0.1: + resolution: {integrity: sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + + dompurify@3.3.1: + resolution: {integrity: sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==} + + emoji-regex-xs@1.0.0: + resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} + + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + + estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + + focus-trap@7.8.0: + resolution: {integrity: sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + hachure-fill@0.5.2: + resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} + + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + + hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + + hookable@5.5.3: + resolution: {integrity: sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ==} + + html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + + iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + + internmap@1.0.1: + resolution: {integrity: sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==} + + internmap@2.0.3: + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} + engines: {node: '>=12'} + + is-what@5.5.0: + resolution: {integrity: sha512-oG7cgbmg5kLYae2N5IVd3jm2s+vldjxJzK1pcu9LfpGuQ93MQSzo0okvRna+7y5ifrD+20FE8FvjusyGaz14fw==} + engines: {node: '>=18'} + + katex@0.16.33: + resolution: {integrity: sha512-q3N5u+1sY9Bu7T4nlXoiRBXWfwSefNGoKeOwekV+gw0cAXQlz2Ww6BLcmBxVDeXBMUDQv6fK5bcNaJLxob3ZQA==} + hasBin: true + + khroma@2.1.0: + resolution: {integrity: sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==} + + langium@4.2.1: + resolution: {integrity: sha512-zu9QWmjpzJcomzdJQAHgDVhLGq5bLosVak1KVa40NzQHXfqr4eAHupvnPOVXEoLkg6Ocefvf/93d//SB7du4YQ==} + engines: {node: '>=20.10.0', npm: '>=10.2.3'} + + layout-base@1.0.2: + resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} + + layout-base@2.0.1: + resolution: {integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==} + + lodash-es@4.17.23: + resolution: {integrity: sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + mark.js@8.11.1: + resolution: {integrity: sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==} + + marked@16.4.2: + resolution: {integrity: sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==} + engines: {node: '>= 20'} + hasBin: true + + mdast-util-to-hast@13.2.1: + resolution: {integrity: sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==} + + mermaid@11.12.3: + resolution: {integrity: sha512-wN5ZSgJQIC+CHJut9xaKWsknLxaFBwCPwPkGTSUYrTiHORWvpT8RxGk849HPnpUAQ+/9BPRqYb80jTpearrHzQ==} + + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + + minisearch@7.2.0: + resolution: {integrity: sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==} + + mitt@3.0.1: + resolution: {integrity: sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==} + + mlly@1.8.0: + resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + non-layered-tidy-tree-layout@2.0.2: + resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==} + + oniguruma-to-es@3.1.1: + resolution: {integrity: sha512-bUH8SDvPkH3ho3dvwJwfonjlQ4R80vjyvrU8YpxuROddv55vAEJrTuCuCVUhhsHbtlD9tGGbaNApGQckXhS8iQ==} + + package-manager-detector@1.6.0: + resolution: {integrity: sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==} + + path-data-parser@0.1.0: + resolution: {integrity: sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + perfect-debounce@1.0.0: + resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + points-on-curve@0.2.0: + resolution: {integrity: sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==} + + points-on-path@0.2.1: + resolution: {integrity: sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==} + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + preact@10.28.4: + resolution: {integrity: sha512-uKFfOHWuSNpRFVTnljsCluEFq57OKT+0QdOiQo8XWnQ/pSvg7OpX5eNOejELXJMWy+BwM2nobz0FkvzmnpCNsQ==} + + property-information@7.1.0: + resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.1.0: + resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + + robust-predicates@3.0.2: + resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} + + rollup@4.59.0: + resolution: {integrity: sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + roughjs@4.6.6: + resolution: {integrity: sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==} + + rw@1.3.3: + resolution: {integrity: sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + search-insights@2.17.3: + resolution: {integrity: sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==} + + shiki@2.5.0: + resolution: {integrity: sha512-mI//trrsaiCIPsja5CNfsyNOqgAZUb6VpJA+340toL42UpzQlXpwRV9nch69X6gaUxrr9kaOOa6e3y3uAkGFxQ==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + + speakingurl@14.0.1: + resolution: {integrity: sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==} + engines: {node: '>=0.10.0'} + + stringify-entities@4.0.4: + resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} + + stylis@4.3.6: + resolution: {integrity: sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==} + + superjson@2.2.6: + resolution: {integrity: sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==} + engines: {node: '>=16'} + + tabbable@6.4.0: + resolution: {integrity: sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==} + + tinyexec@1.0.2: + resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} + engines: {node: '>=18'} + + trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + + ts-dedent@2.2.0: + resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==} + engines: {node: '>=6.10'} + + ufo@1.6.3: + resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} + + unist-util-is@6.0.1: + resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} + + unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + + unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + + unist-util-visit-parents@6.0.2: + resolution: {integrity: sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==} + + unist-util-visit@5.1.0: + resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} + + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + vfile-message@4.0.3: + resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==} + + vfile@6.0.3: + resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + + vitepress-plugin-mermaid@2.0.17: + resolution: {integrity: sha512-IUzYpwf61GC6k0XzfmAmNrLvMi9TRrVRMsUyCA8KNXhg/mQ1VqWnO0/tBVPiX5UoKF1mDUwqn5QV4qAJl6JnUg==} + peerDependencies: + mermaid: 10 || 11 + vitepress: ^1.0.0 || ^1.0.0-alpha + + vitepress@1.6.4: + resolution: {integrity: sha512-+2ym1/+0VVrbhNyRoFFesVvBvHAVMZMK0rw60E3X/5349M1GuVdKeazuksqopEdvkKwKGs21Q729jX81/bkBJg==} + hasBin: true + peerDependencies: + markdown-it-mathjax3: ^4 + postcss: ^8 + peerDependenciesMeta: + markdown-it-mathjax3: + optional: true + postcss: + optional: true + + vscode-jsonrpc@8.2.0: + resolution: {integrity: sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==} + engines: {node: '>=14.0.0'} + + vscode-languageserver-protocol@3.17.5: + resolution: {integrity: sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==} + + vscode-languageserver-textdocument@1.0.12: + resolution: {integrity: sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==} + + vscode-languageserver-types@3.17.5: + resolution: {integrity: sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==} + + vscode-languageserver@9.0.1: + resolution: {integrity: sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==} + hasBin: true + + vscode-uri@3.1.0: + resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} + + vue@3.5.29: + resolution: {integrity: sha512-BZqN4Ze6mDQVNAni0IHeMJ5mwr8VAJ3MQC9FmprRhcBYENw+wOAAjRj8jfmN6FLl0j96OXbR+CjWhmAmM+QGnA==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + + zwitch@2.0.4: + resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + +snapshots: + + '@algolia/abtesting@1.15.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/autocomplete-core@1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)(search-insights@2.17.3)': + dependencies: + '@algolia/autocomplete-plugin-algolia-insights': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)(search-insights@2.17.3) + '@algolia/autocomplete-shared': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1) + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + - search-insights + + '@algolia/autocomplete-plugin-algolia-insights@1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)(search-insights@2.17.3)': + dependencies: + '@algolia/autocomplete-shared': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1) + search-insights: 2.17.3 + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + + '@algolia/autocomplete-preset-algolia@1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)': + dependencies: + '@algolia/autocomplete-shared': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1) + '@algolia/client-search': 5.49.1 + algoliasearch: 5.49.1 + + '@algolia/autocomplete-shared@1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)': + dependencies: + '@algolia/client-search': 5.49.1 + algoliasearch: 5.49.1 + + '@algolia/client-abtesting@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/client-analytics@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/client-common@5.49.1': {} + + '@algolia/client-insights@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/client-personalization@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/client-query-suggestions@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/client-search@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/ingestion@1.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/monitoring@1.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/recommend@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + '@algolia/requester-browser-xhr@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + + '@algolia/requester-fetch@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + + '@algolia/requester-node-http@5.49.1': + dependencies: + '@algolia/client-common': 5.49.1 + + '@antfu/install-pkg@1.1.0': + dependencies: + package-manager-detector: 1.6.0 + tinyexec: 1.0.2 + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/parser@7.29.0': + dependencies: + '@babel/types': 7.29.0 + + '@babel/types@7.29.0': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@braintree/sanitize-url@6.0.4': + optional: true + + '@braintree/sanitize-url@7.1.2': {} + + '@chevrotain/cst-dts-gen@11.1.2': + dependencies: + '@chevrotain/gast': 11.1.2 + '@chevrotain/types': 11.1.2 + lodash-es: 4.17.23 + + '@chevrotain/gast@11.1.2': + dependencies: + '@chevrotain/types': 11.1.2 + lodash-es: 4.17.23 + + '@chevrotain/regexp-to-ast@11.1.2': {} + + '@chevrotain/types@11.1.2': {} + + '@chevrotain/utils@11.1.2': {} + + '@docsearch/css@3.8.2': {} + + '@docsearch/js@3.8.2(@algolia/client-search@5.49.1)(search-insights@2.17.3)': + dependencies: + '@docsearch/react': 3.8.2(@algolia/client-search@5.49.1)(search-insights@2.17.3) + preact: 10.28.4 + transitivePeerDependencies: + - '@algolia/client-search' + - '@types/react' + - react + - react-dom + - search-insights + + '@docsearch/react@3.8.2(@algolia/client-search@5.49.1)(search-insights@2.17.3)': + dependencies: + '@algolia/autocomplete-core': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1)(search-insights@2.17.3) + '@algolia/autocomplete-preset-algolia': 1.17.7(@algolia/client-search@5.49.1)(algoliasearch@5.49.1) + '@docsearch/css': 3.8.2 + algoliasearch: 5.49.1 + optionalDependencies: + search-insights: 2.17.3 + transitivePeerDependencies: + - '@algolia/client-search' + + '@esbuild/aix-ppc64@0.21.5': + optional: true + + '@esbuild/android-arm64@0.21.5': + optional: true + + '@esbuild/android-arm@0.21.5': + optional: true + + '@esbuild/android-x64@0.21.5': + optional: true + + '@esbuild/darwin-arm64@0.21.5': + optional: true + + '@esbuild/darwin-x64@0.21.5': + optional: true + + '@esbuild/freebsd-arm64@0.21.5': + optional: true + + '@esbuild/freebsd-x64@0.21.5': + optional: true + + '@esbuild/linux-arm64@0.21.5': + optional: true + + '@esbuild/linux-arm@0.21.5': + optional: true + + '@esbuild/linux-ia32@0.21.5': + optional: true + + '@esbuild/linux-loong64@0.21.5': + optional: true + + '@esbuild/linux-mips64el@0.21.5': + optional: true + + '@esbuild/linux-ppc64@0.21.5': + optional: true + + '@esbuild/linux-riscv64@0.21.5': + optional: true + + '@esbuild/linux-s390x@0.21.5': + optional: true + + '@esbuild/linux-x64@0.21.5': + optional: true + + '@esbuild/netbsd-x64@0.21.5': + optional: true + + '@esbuild/openbsd-x64@0.21.5': + optional: true + + '@esbuild/sunos-x64@0.21.5': + optional: true + + '@esbuild/win32-arm64@0.21.5': + optional: true + + '@esbuild/win32-ia32@0.21.5': + optional: true + + '@esbuild/win32-x64@0.21.5': + optional: true + + '@iconify-json/simple-icons@1.2.71': + dependencies: + '@iconify/types': 2.0.0 + + '@iconify/types@2.0.0': {} + + '@iconify/utils@3.1.0': + dependencies: + '@antfu/install-pkg': 1.1.0 + '@iconify/types': 2.0.0 + mlly: 1.8.0 + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@mermaid-js/mermaid-mindmap@9.3.0': + dependencies: + '@braintree/sanitize-url': 6.0.4 + cytoscape: 3.33.1 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.33.1) + cytoscape-fcose: 2.2.0(cytoscape@3.33.1) + d3: 7.9.0 + khroma: 2.1.0 + non-layered-tidy-tree-layout: 2.0.2 + optional: true + + '@mermaid-js/parser@1.0.0': + dependencies: + langium: 4.2.1 + + '@rollup/rollup-android-arm-eabi@4.59.0': + optional: true + + '@rollup/rollup-android-arm64@4.59.0': + optional: true + + '@rollup/rollup-darwin-arm64@4.59.0': + optional: true + + '@rollup/rollup-darwin-x64@4.59.0': + optional: true + + '@rollup/rollup-freebsd-arm64@4.59.0': + optional: true + + '@rollup/rollup-freebsd-x64@4.59.0': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.59.0': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-x64-musl@4.59.0': + optional: true + + '@rollup/rollup-openbsd-x64@4.59.0': + optional: true + + '@rollup/rollup-openharmony-arm64@4.59.0': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.59.0': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.59.0': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.59.0': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.59.0': + optional: true + + '@shikijs/core@2.5.0': + dependencies: + '@shikijs/engine-javascript': 2.5.0 + '@shikijs/engine-oniguruma': 2.5.0 + '@shikijs/types': 2.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@2.5.0': + dependencies: + '@shikijs/types': 2.5.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 3.1.1 + + '@shikijs/engine-oniguruma@2.5.0': + dependencies: + '@shikijs/types': 2.5.0 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@2.5.0': + dependencies: + '@shikijs/types': 2.5.0 + + '@shikijs/themes@2.5.0': + dependencies: + '@shikijs/types': 2.5.0 + + '@shikijs/transformers@2.5.0': + dependencies: + '@shikijs/core': 2.5.0 + '@shikijs/types': 2.5.0 + + '@shikijs/types@2.5.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + + '@shikijs/vscode-textmate@10.0.2': {} + + '@types/d3-array@3.2.2': {} + + '@types/d3-axis@3.0.6': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-brush@3.0.6': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-chord@3.0.6': {} + + '@types/d3-color@3.1.3': {} + + '@types/d3-contour@3.0.6': + dependencies: + '@types/d3-array': 3.2.2 + '@types/geojson': 7946.0.16 + + '@types/d3-delaunay@6.0.4': {} + + '@types/d3-dispatch@3.0.7': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-dsv@3.0.7': {} + + '@types/d3-ease@3.0.2': {} + + '@types/d3-fetch@3.0.7': + dependencies: + '@types/d3-dsv': 3.0.7 + + '@types/d3-force@3.0.10': {} + + '@types/d3-format@3.0.4': {} + + '@types/d3-geo@3.1.0': + dependencies: + '@types/geojson': 7946.0.16 + + '@types/d3-hierarchy@3.1.7': {} + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-path@3.1.1': {} + + '@types/d3-polygon@3.0.2': {} + + '@types/d3-quadtree@3.0.6': {} + + '@types/d3-random@3.0.3': {} + + '@types/d3-scale-chromatic@3.1.0': {} + + '@types/d3-scale@4.0.9': + dependencies: + '@types/d3-time': 3.0.4 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-shape@3.1.8': + dependencies: + '@types/d3-path': 3.1.1 + + '@types/d3-time-format@4.0.3': {} + + '@types/d3-time@3.0.4': {} + + '@types/d3-timer@3.0.2': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + + '@types/d3@7.4.3': + dependencies: + '@types/d3-array': 3.2.2 + '@types/d3-axis': 3.0.6 + '@types/d3-brush': 3.0.6 + '@types/d3-chord': 3.0.6 + '@types/d3-color': 3.1.3 + '@types/d3-contour': 3.0.6 + '@types/d3-delaunay': 6.0.4 + '@types/d3-dispatch': 3.0.7 + '@types/d3-drag': 3.0.7 + '@types/d3-dsv': 3.0.7 + '@types/d3-ease': 3.0.2 + '@types/d3-fetch': 3.0.7 + '@types/d3-force': 3.0.10 + '@types/d3-format': 3.0.4 + '@types/d3-geo': 3.1.0 + '@types/d3-hierarchy': 3.1.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-path': 3.1.1 + '@types/d3-polygon': 3.0.2 + '@types/d3-quadtree': 3.0.6 + '@types/d3-random': 3.0.3 + '@types/d3-scale': 4.0.9 + '@types/d3-scale-chromatic': 3.1.0 + '@types/d3-selection': 3.0.11 + '@types/d3-shape': 3.1.8 + '@types/d3-time': 3.0.4 + '@types/d3-time-format': 4.0.3 + '@types/d3-timer': 3.0.2 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + + '@types/estree@1.0.8': {} + + '@types/geojson@7946.0.16': {} + + '@types/hast@3.0.4': + dependencies: + '@types/unist': 3.0.3 + + '@types/linkify-it@5.0.0': {} + + '@types/markdown-it@14.1.2': + dependencies: + '@types/linkify-it': 5.0.0 + '@types/mdurl': 2.0.0 + + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + + '@types/mdurl@2.0.0': {} + + '@types/trusted-types@2.0.7': + optional: true + + '@types/unist@3.0.3': {} + + '@types/web-bluetooth@0.0.21': {} + + '@ungap/structured-clone@1.3.0': {} + + '@vitejs/plugin-vue@5.2.4(vite@5.4.21)(vue@3.5.29)': + dependencies: + vite: 5.4.21 + vue: 3.5.29 + + '@vue/compiler-core@3.5.29': + dependencies: + '@babel/parser': 7.29.0 + '@vue/shared': 3.5.29 + entities: 7.0.1 + estree-walker: 2.0.2 + source-map-js: 1.2.1 + + '@vue/compiler-dom@3.5.29': + dependencies: + '@vue/compiler-core': 3.5.29 + '@vue/shared': 3.5.29 + + '@vue/compiler-sfc@3.5.29': + dependencies: + '@babel/parser': 7.29.0 + '@vue/compiler-core': 3.5.29 + '@vue/compiler-dom': 3.5.29 + '@vue/compiler-ssr': 3.5.29 + '@vue/shared': 3.5.29 + estree-walker: 2.0.2 + magic-string: 0.30.21 + postcss: 8.5.6 + source-map-js: 1.2.1 + + '@vue/compiler-ssr@3.5.29': + dependencies: + '@vue/compiler-dom': 3.5.29 + '@vue/shared': 3.5.29 + + '@vue/devtools-api@7.7.9': + dependencies: + '@vue/devtools-kit': 7.7.9 + + '@vue/devtools-kit@7.7.9': + dependencies: + '@vue/devtools-shared': 7.7.9 + birpc: 2.9.0 + hookable: 5.5.3 + mitt: 3.0.1 + perfect-debounce: 1.0.0 + speakingurl: 14.0.1 + superjson: 2.2.6 + + '@vue/devtools-shared@7.7.9': + dependencies: + rfdc: 1.4.1 + + '@vue/reactivity@3.5.29': + dependencies: + '@vue/shared': 3.5.29 + + '@vue/runtime-core@3.5.29': + dependencies: + '@vue/reactivity': 3.5.29 + '@vue/shared': 3.5.29 + + '@vue/runtime-dom@3.5.29': + dependencies: + '@vue/reactivity': 3.5.29 + '@vue/runtime-core': 3.5.29 + '@vue/shared': 3.5.29 + csstype: 3.2.3 + + '@vue/server-renderer@3.5.29(vue@3.5.29)': + dependencies: + '@vue/compiler-ssr': 3.5.29 + '@vue/shared': 3.5.29 + vue: 3.5.29 + + '@vue/shared@3.5.29': {} + + '@vueuse/core@12.8.2': + dependencies: + '@types/web-bluetooth': 0.0.21 + '@vueuse/metadata': 12.8.2 + '@vueuse/shared': 12.8.2 + vue: 3.5.29 + transitivePeerDependencies: + - typescript + + '@vueuse/integrations@12.8.2(focus-trap@7.8.0)': + dependencies: + '@vueuse/core': 12.8.2 + '@vueuse/shared': 12.8.2 + vue: 3.5.29 + optionalDependencies: + focus-trap: 7.8.0 + transitivePeerDependencies: + - typescript + + '@vueuse/metadata@12.8.2': {} + + '@vueuse/shared@12.8.2': + dependencies: + vue: 3.5.29 + transitivePeerDependencies: + - typescript + + acorn@8.16.0: {} + + algoliasearch@5.49.1: + dependencies: + '@algolia/abtesting': 1.15.1 + '@algolia/client-abtesting': 5.49.1 + '@algolia/client-analytics': 5.49.1 + '@algolia/client-common': 5.49.1 + '@algolia/client-insights': 5.49.1 + '@algolia/client-personalization': 5.49.1 + '@algolia/client-query-suggestions': 5.49.1 + '@algolia/client-search': 5.49.1 + '@algolia/ingestion': 1.49.1 + '@algolia/monitoring': 1.49.1 + '@algolia/recommend': 5.49.1 + '@algolia/requester-browser-xhr': 5.49.1 + '@algolia/requester-fetch': 5.49.1 + '@algolia/requester-node-http': 5.49.1 + + birpc@2.9.0: {} + + ccount@2.0.1: {} + + character-entities-html4@2.1.0: {} + + character-entities-legacy@3.0.0: {} + + chevrotain-allstar@0.3.1(chevrotain@11.1.2): + dependencies: + chevrotain: 11.1.2 + lodash-es: 4.17.23 + + chevrotain@11.1.2: + dependencies: + '@chevrotain/cst-dts-gen': 11.1.2 + '@chevrotain/gast': 11.1.2 + '@chevrotain/regexp-to-ast': 11.1.2 + '@chevrotain/types': 11.1.2 + '@chevrotain/utils': 11.1.2 + lodash-es: 4.17.23 + + comma-separated-tokens@2.0.3: {} + + commander@7.2.0: {} + + commander@8.3.0: {} + + confbox@0.1.8: {} + + copy-anything@4.0.5: + dependencies: + is-what: 5.5.0 + + cose-base@1.0.3: + dependencies: + layout-base: 1.0.2 + + cose-base@2.2.0: + dependencies: + layout-base: 2.0.1 + + csstype@3.2.3: {} + + cytoscape-cose-bilkent@4.1.0(cytoscape@3.33.1): + dependencies: + cose-base: 1.0.3 + cytoscape: 3.33.1 + + cytoscape-fcose@2.2.0(cytoscape@3.33.1): + dependencies: + cose-base: 2.2.0 + cytoscape: 3.33.1 + + cytoscape@3.33.1: {} + + d3-array@2.12.1: + dependencies: + internmap: 1.0.1 + + d3-array@3.2.4: + dependencies: + internmap: 2.0.3 + + d3-axis@3.0.0: {} + + d3-brush@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + + d3-chord@3.0.1: + dependencies: + d3-path: 3.1.0 + + d3-color@3.1.0: {} + + d3-contour@4.0.2: + dependencies: + d3-array: 3.2.4 + + d3-delaunay@6.0.4: + dependencies: + delaunator: 5.0.1 + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-dsv@3.0.1: + dependencies: + commander: 7.2.0 + iconv-lite: 0.6.3 + rw: 1.3.3 + + d3-ease@3.0.1: {} + + d3-fetch@3.0.1: + dependencies: + d3-dsv: 3.0.1 + + d3-force@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-quadtree: 3.0.1 + d3-timer: 3.0.1 + + d3-format@3.1.2: {} + + d3-geo@3.1.1: + dependencies: + d3-array: 3.2.4 + + d3-hierarchy@3.1.2: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-path@1.0.9: {} + + d3-path@3.1.0: {} + + d3-polygon@3.0.1: {} + + d3-quadtree@3.0.1: {} + + d3-random@3.0.1: {} + + d3-sankey@0.12.3: + dependencies: + d3-array: 2.12.1 + d3-shape: 1.3.7 + + d3-scale-chromatic@3.1.0: + dependencies: + d3-color: 3.1.0 + d3-interpolate: 3.0.1 + + d3-scale@4.0.2: + dependencies: + d3-array: 3.2.4 + d3-format: 3.1.2 + d3-interpolate: 3.0.1 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + + d3-selection@3.0.0: {} + + d3-shape@1.3.7: + dependencies: + d3-path: 1.0.9 + + d3-shape@3.2.0: + dependencies: + d3-path: 3.1.0 + + d3-time-format@4.1.0: + dependencies: + d3-time: 3.1.0 + + d3-time@3.1.0: + dependencies: + d3-array: 3.2.4 + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + + d3@7.9.0: + dependencies: + d3-array: 3.2.4 + d3-axis: 3.0.0 + d3-brush: 3.0.0 + d3-chord: 3.0.1 + d3-color: 3.1.0 + d3-contour: 4.0.2 + d3-delaunay: 6.0.4 + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-dsv: 3.0.1 + d3-ease: 3.0.1 + d3-fetch: 3.0.1 + d3-force: 3.0.0 + d3-format: 3.1.2 + d3-geo: 3.1.1 + d3-hierarchy: 3.1.2 + d3-interpolate: 3.0.1 + d3-path: 3.1.0 + d3-polygon: 3.0.1 + d3-quadtree: 3.0.1 + d3-random: 3.0.1 + d3-scale: 4.0.2 + d3-scale-chromatic: 3.1.0 + d3-selection: 3.0.0 + d3-shape: 3.2.0 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + d3-timer: 3.0.1 + d3-transition: 3.0.1(d3-selection@3.0.0) + d3-zoom: 3.0.0 + + dagre-d3-es@7.0.13: + dependencies: + d3: 7.9.0 + lodash-es: 4.17.23 + + dayjs@1.11.19: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + delaunator@5.0.1: + dependencies: + robust-predicates: 3.0.2 + + dequal@2.0.3: {} + + devlop@1.1.0: + dependencies: + dequal: 2.0.3 + + dompurify@3.3.1: + optionalDependencies: + '@types/trusted-types': 2.0.7 + + emoji-regex-xs@1.0.0: {} + + entities@7.0.1: {} + + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + + estree-walker@2.0.2: {} + + focus-trap@7.8.0: + dependencies: + tabbable: 6.4.0 + + fsevents@2.3.3: + optional: true + + hachure-fill@0.5.2: {} + + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.1 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + + hast-util-whitespace@3.0.0: + dependencies: + '@types/hast': 3.0.4 + + hookable@5.5.3: {} + + html-void-elements@3.0.0: {} + + iconv-lite@0.6.3: + dependencies: + safer-buffer: 2.1.2 + + internmap@1.0.1: {} + + internmap@2.0.3: {} + + is-what@5.5.0: {} + + katex@0.16.33: + dependencies: + commander: 8.3.0 + + khroma@2.1.0: {} + + langium@4.2.1: + dependencies: + chevrotain: 11.1.2 + chevrotain-allstar: 0.3.1(chevrotain@11.1.2) + vscode-languageserver: 9.0.1 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + + layout-base@1.0.2: {} + + layout-base@2.0.1: {} + + lodash-es@4.17.23: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + mark.js@8.11.1: {} + + marked@16.4.2: {} + + mdast-util-to-hast@13.2.1: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.1 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + + mermaid@11.12.3: + dependencies: + '@braintree/sanitize-url': 7.1.2 + '@iconify/utils': 3.1.0 + '@mermaid-js/parser': 1.0.0 + '@types/d3': 7.4.3 + cytoscape: 3.33.1 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.33.1) + cytoscape-fcose: 2.2.0(cytoscape@3.33.1) + d3: 7.9.0 + d3-sankey: 0.12.3 + dagre-d3-es: 7.0.13 + dayjs: 1.11.19 + dompurify: 3.3.1 + katex: 0.16.33 + khroma: 2.1.0 + lodash-es: 4.17.23 + marked: 16.4.2 + roughjs: 4.6.6 + stylis: 4.3.6 + ts-dedent: 2.2.0 + uuid: 11.1.0 + + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-util-encode@2.0.1: {} + + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + + micromark-util-symbol@2.0.1: {} + + micromark-util-types@2.0.2: {} + + minisearch@7.2.0: {} + + mitt@3.0.1: {} + + mlly@1.8.0: + dependencies: + acorn: 8.16.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.3 + + ms@2.1.3: {} + + nanoid@3.3.11: {} + + non-layered-tidy-tree-layout@2.0.2: + optional: true + + oniguruma-to-es@3.1.1: + dependencies: + emoji-regex-xs: 1.0.0 + regex: 6.1.0 + regex-recursion: 6.0.2 + + package-manager-detector@1.6.0: {} + + path-data-parser@0.1.0: {} + + pathe@2.0.3: {} + + perfect-debounce@1.0.0: {} + + picocolors@1.1.1: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.0 + pathe: 2.0.3 + + points-on-curve@0.2.0: {} + + points-on-path@0.2.1: + dependencies: + path-data-parser: 0.1.0 + points-on-curve: 0.2.0 + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + preact@10.28.4: {} + + property-information@7.1.0: {} + + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.1.0: + dependencies: + regex-utilities: 2.3.0 + + rfdc@1.4.1: {} + + robust-predicates@3.0.2: {} + + rollup@4.59.0: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.59.0 + '@rollup/rollup-android-arm64': 4.59.0 + '@rollup/rollup-darwin-arm64': 4.59.0 + '@rollup/rollup-darwin-x64': 4.59.0 + '@rollup/rollup-freebsd-arm64': 4.59.0 + '@rollup/rollup-freebsd-x64': 4.59.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.59.0 + '@rollup/rollup-linux-arm-musleabihf': 4.59.0 + '@rollup/rollup-linux-arm64-gnu': 4.59.0 + '@rollup/rollup-linux-arm64-musl': 4.59.0 + '@rollup/rollup-linux-loong64-gnu': 4.59.0 + '@rollup/rollup-linux-loong64-musl': 4.59.0 + '@rollup/rollup-linux-ppc64-gnu': 4.59.0 + '@rollup/rollup-linux-ppc64-musl': 4.59.0 + '@rollup/rollup-linux-riscv64-gnu': 4.59.0 + '@rollup/rollup-linux-riscv64-musl': 4.59.0 + '@rollup/rollup-linux-s390x-gnu': 4.59.0 + '@rollup/rollup-linux-x64-gnu': 4.59.0 + '@rollup/rollup-linux-x64-musl': 4.59.0 + '@rollup/rollup-openbsd-x64': 4.59.0 + '@rollup/rollup-openharmony-arm64': 4.59.0 + '@rollup/rollup-win32-arm64-msvc': 4.59.0 + '@rollup/rollup-win32-ia32-msvc': 4.59.0 + '@rollup/rollup-win32-x64-gnu': 4.59.0 + '@rollup/rollup-win32-x64-msvc': 4.59.0 + fsevents: 2.3.3 + + roughjs@4.6.6: + dependencies: + hachure-fill: 0.5.2 + path-data-parser: 0.1.0 + points-on-curve: 0.2.0 + points-on-path: 0.2.1 + + rw@1.3.3: {} + + safer-buffer@2.1.2: {} + + search-insights@2.17.3: {} + + shiki@2.5.0: + dependencies: + '@shikijs/core': 2.5.0 + '@shikijs/engine-javascript': 2.5.0 + '@shikijs/engine-oniguruma': 2.5.0 + '@shikijs/langs': 2.5.0 + '@shikijs/themes': 2.5.0 + '@shikijs/types': 2.5.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + + source-map-js@1.2.1: {} + + space-separated-tokens@2.0.2: {} + + speakingurl@14.0.1: {} + + stringify-entities@4.0.4: + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + + stylis@4.3.6: {} + + superjson@2.2.6: + dependencies: + copy-anything: 4.0.5 + + tabbable@6.4.0: {} + + tinyexec@1.0.2: {} + + trim-lines@3.0.1: {} + + ts-dedent@2.2.0: {} + + ufo@1.6.3: {} + + unist-util-is@6.0.1: + dependencies: + '@types/unist': 3.0.3 + + unist-util-position@5.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-stringify-position@4.0.0: + dependencies: + '@types/unist': 3.0.3 + + unist-util-visit-parents@6.0.2: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + + unist-util-visit@5.1.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.1 + unist-util-visit-parents: 6.0.2 + + uuid@11.1.0: {} + + vfile-message@4.0.3: + dependencies: + '@types/unist': 3.0.3 + unist-util-stringify-position: 4.0.0 + + vfile@6.0.3: + dependencies: + '@types/unist': 3.0.3 + vfile-message: 4.0.3 + + vite@5.4.21: + dependencies: + esbuild: 0.21.5 + postcss: 8.5.6 + rollup: 4.59.0 + optionalDependencies: + fsevents: 2.3.3 + + vitepress-plugin-mermaid@2.0.17(mermaid@11.12.3)(vitepress@1.6.4(@algolia/client-search@5.49.1)(postcss@8.5.6)(search-insights@2.17.3)): + dependencies: + mermaid: 11.12.3 + vitepress: 1.6.4(@algolia/client-search@5.49.1)(postcss@8.5.6)(search-insights@2.17.3) + optionalDependencies: + '@mermaid-js/mermaid-mindmap': 9.3.0 + + vitepress@1.6.4(@algolia/client-search@5.49.1)(postcss@8.5.6)(search-insights@2.17.3): + dependencies: + '@docsearch/css': 3.8.2 + '@docsearch/js': 3.8.2(@algolia/client-search@5.49.1)(search-insights@2.17.3) + '@iconify-json/simple-icons': 1.2.71 + '@shikijs/core': 2.5.0 + '@shikijs/transformers': 2.5.0 + '@shikijs/types': 2.5.0 + '@types/markdown-it': 14.1.2 + '@vitejs/plugin-vue': 5.2.4(vite@5.4.21)(vue@3.5.29) + '@vue/devtools-api': 7.7.9 + '@vue/shared': 3.5.29 + '@vueuse/core': 12.8.2 + '@vueuse/integrations': 12.8.2(focus-trap@7.8.0) + focus-trap: 7.8.0 + mark.js: 8.11.1 + minisearch: 7.2.0 + shiki: 2.5.0 + vite: 5.4.21 + vue: 3.5.29 + optionalDependencies: + postcss: 8.5.6 + transitivePeerDependencies: + - '@algolia/client-search' + - '@types/node' + - '@types/react' + - async-validator + - axios + - change-case + - drauu + - fuse.js + - idb-keyval + - jwt-decode + - less + - lightningcss + - nprogress + - qrcode + - react + - react-dom + - sass + - sass-embedded + - search-insights + - sortablejs + - stylus + - sugarss + - terser + - typescript + - universal-cookie + + vscode-jsonrpc@8.2.0: {} + + vscode-languageserver-protocol@3.17.5: + dependencies: + vscode-jsonrpc: 8.2.0 + vscode-languageserver-types: 3.17.5 + + vscode-languageserver-textdocument@1.0.12: {} + + vscode-languageserver-types@3.17.5: {} + + vscode-languageserver@9.0.1: + dependencies: + vscode-languageserver-protocol: 3.17.5 + + vscode-uri@3.1.0: {} + + vue@3.5.29: + dependencies: + '@vue/compiler-dom': 3.5.29 + '@vue/compiler-sfc': 3.5.29 + '@vue/runtime-dom': 3.5.29 + '@vue/server-renderer': 3.5.29(vue@3.5.29) + '@vue/shared': 3.5.29 + + zwitch@2.0.4: {} diff --git a/docs/reference/config-example.md b/docs/reference/config-example.md new file mode 100644 index 0000000..b97b264 --- /dev/null +++ b/docs/reference/config-example.md @@ -0,0 +1,164 @@ +# Configuration Example + +A complete, annotated configuration file showing all available options. + +```toml +# ============================================================================= +# Virtual Buckets +# ============================================================================= + +# A publicly accessible S3 bucket (anonymous reads allowed) +[[buckets]] +name = "public-data" # Client-visible bucket name +backend_type = "s3" # Backend provider: "s3", "az", or "gcs" +anonymous_access = true # Allow GET/HEAD/LIST without auth +allowed_roles = [] # No STS roles (anonymous only) + +[buckets.backend_options] +endpoint = "https://s3.us-east-1.amazonaws.com" +bucket_name = "my-company-public-assets" # Actual backend bucket name +region = "us-east-1" +access_key_id = "AKIAIOSFODNN7EXAMPLE" +secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +# A private S3 bucket backed by MinIO with a backend prefix +[[buckets]] +name = "ml-artifacts" +backend_type = "s3" +backend_prefix = "v2" # Prepend "v2/" to all keys when forwarding +anonymous_access = false +allowed_roles = ["github-actions-deployer"] + +[buckets.backend_options] +endpoint = "https://minio.internal:9000" +bucket_name = "ml-pipeline-artifacts" +region = "us-east-1" +access_key_id = "minioadmin" +secret_access_key = "minioadmin" + +# An S3 bucket on a different region +[[buckets]] +name = "deploy-bundles" +backend_type = "s3" +anonymous_access = false +allowed_roles = ["github-actions-deployer", "ci-readonly"] + +[buckets.backend_options] +endpoint = "https://s3.us-west-2.amazonaws.com" +bucket_name = "prod-deploy-bundles" +region = "us-west-2" +access_key_id = "AKIAI44QH8DHBEXAMPLE" +secret_access_key = "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + +# An Azure Blob Storage backend (requires "azure" feature) +[[buckets]] +name = "azure-data" +backend_type = "az" +anonymous_access = true +allowed_roles = [] + +[buckets.backend_options] +account_name = "mystorageaccount" +container_name = "public-datasets" + +# ============================================================================= +# IAM Roles (for STS AssumeRoleWithWebIdentity) +# ============================================================================= + +# Role for GitHub Actions CI/CD pipelines +[[roles]] +role_id = "github-actions-deployer" # Used as RoleArn in STS requests +name = "GitHub Actions Deploy Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +required_audience = "sts.s3proxy.example.com" # Token's `aud` must match + +# Glob patterns for the `sub` claim +subject_conditions = [ + "repo:myorg/myapp:ref:refs/heads/main", + "repo:myorg/myapp:ref:refs/heads/release/*", + "repo:myorg/infrastructure:*", +] +max_session_duration_secs = 3600 # 1 hour max + +# Scopes granted to minted credentials +[[roles.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/", "datasets/"] # Restrict to these prefixes +actions = [ + "get_object", "head_object", "put_object", + "create_multipart_upload", "upload_part", "complete_multipart_upload" +] + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] # Full bucket access +actions = [ + "get_object", "head_object", "put_object", + "create_multipart_upload", "upload_part", "complete_multipart_upload" +] + +# Role with template variables for per-user access +[[roles]] +role_id = "source-user" +name = "Source Cooperative User" +trusted_oidc_issuers = ["https://auth.source.coop", "https://auth.staging.source.coop"] +subject_conditions = ["*"] # Any subject +max_session_duration_secs = 3600 + +# {sub} is replaced with the JWT's `sub` claim at mint time +[[roles.allowed_scopes]] +bucket = "{sub}" +prefixes = [] +actions = ["get_object", "head_object", "put_object", "list_bucket"] + +# Read-only role for CI +[[roles]] +role_id = "ci-readonly" +name = "CI Read-Only Role" +trusted_oidc_issuers = ["https://token.actions.githubusercontent.com"] +subject_conditions = ["repo:myorg/*"] # Any repo in the org +max_session_duration_secs = 1800 # 30 minutes + +[[roles.allowed_scopes]] +bucket = "deploy-bundles" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] + +# ============================================================================= +# Long-Lived Credentials +# ============================================================================= + +# Service account for an internal tool +[[credentials]] +access_key_id = "AKPROXY00000EXAMPLE" +secret_access_key = "proxy/secret/key/EXAMPLE000000000000" +principal_name = "internal-dashboard" +created_at = "2024-01-15T00:00:00Z" +enabled = true # Set to false to revoke + +[[credentials.allowed_scopes]] +bucket = "public-data" +prefixes = [] +actions = ["get_object", "head_object", "list_bucket"] + +[[credentials.allowed_scopes]] +bucket = "ml-artifacts" +prefixes = ["models/production/"] +actions = ["get_object", "head_object"] +``` + +## Environment Variables + +These are set separately from the config file: + +```bash +# Required for STS temporary credentials (sealed tokens) +export SESSION_TOKEN_KEY=$(openssl rand -base64 32) + +# Required for OIDC backend auth +export OIDC_PROVIDER_KEY=$(cat oidc-key.pem) +export OIDC_PROVIDER_ISSUER="https://data.source.coop" + +# Logging +export RUST_LOG="source_coop=info" +``` diff --git a/docs/reference/errors.md b/docs/reference/errors.md new file mode 100644 index 0000000..569fe9e --- /dev/null +++ b/docs/reference/errors.md @@ -0,0 +1,58 @@ +# Error Codes + +The proxy returns S3-compatible error responses in XML format: + +```xml + + AccessDenied + Access Denied + 550e8400-e29b-41d4-a716-446655440000 + +``` + +## Error Types + +| Error | HTTP Status | S3 Code | When | +|-------|------------|---------|------| +| BucketNotFound | 404 | `NoSuchBucket` | Requested bucket doesn't exist in config | +| NoSuchKey | 404 | `NoSuchKey` | Key not found in backend (forwarded from backend response) | +| AccessDenied | 403 | `AccessDenied` | Caller lacks permission for the requested operation | +| SignatureDoesNotMatch | 403 | `SignatureDoesNotMatch` | SigV4 signature verification failed | +| MissingAuth | 403 | `AccessDenied` | Authentication required but no credentials provided | +| ExpiredCredentials | 403 | `ExpiredToken` | Temporary credentials have expired | +| InvalidOidcToken | 400 | `InvalidIdentityToken` | JWT validation failed (bad signature, untrusted issuer, etc.) | +| RoleNotFound | 403 | `AccessDenied` | Requested role doesn't exist in config | +| InvalidRequest | 400 | `InvalidRequest` | Malformed S3 request | +| BackendError | 503 | `ServiceUnavailable` | Backend object store is unreachable or returned an error | +| PreconditionFailed | 412 | `PreconditionFailed` | Conditional request failed (If-Match, etc.) | +| NotModified | 304 | `NotModified` | Conditional request — content not changed | +| ConfigError | 500 | `InternalError` | Invalid proxy configuration | +| Internal | 500 | `InternalError` | Unexpected internal error | + +## STS Error Responses + +STS errors follow the AWS STS error format: + +```xml + + + InvalidIdentityToken + Token signature verification failed + + 550e8400-e29b-41d4-a716-446655440000 + +``` + +| HTTP Status | Code | When | +|------------|------|------| +| 400 | `MalformedPolicyDocument` | Role not found in config | +| 400 | `InvalidIdentityToken` | JWT invalid, untrusted issuer, algorithm unsupported, subject mismatch | +| 400 | `InvalidParameterValue` | Missing required STS parameters | +| 403 | `AccessDenied` | General authorization failure | +| 500 | `InternalError` | Unexpected error during token exchange | + +## Error Message Safety + +For 5xx errors, the proxy returns generic messages to avoid leaking internal infrastructure details. The full error message is logged server-side but not exposed to clients. + +For 4xx errors, the proxy returns descriptive messages to help clients debug authentication and authorization issues. diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 0000000..0ea31a6 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,5 @@ +# Reference + +- [Supported Operations](./operations) — S3 operations the proxy handles, including dispatch method +- [Error Codes](./errors) — Error types, S3 error codes, and HTTP status codes +- [Config Example](./config-example) — Annotated full configuration file diff --git a/docs/reference/operations.md b/docs/reference/operations.md new file mode 100644 index 0000000..09388e8 --- /dev/null +++ b/docs/reference/operations.md @@ -0,0 +1,43 @@ +# Supported Operations + +## S3 Operations + +| Operation | HTTP Method | Dispatch | Description | +|-----------|------------|----------|-------------| +| GetObject | `GET /{bucket}/{key}` | Forward | Download a file | +| HeadObject | `HEAD /{bucket}/{key}` | Forward | Get file metadata | +| PutObject | `PUT /{bucket}/{key}` | Forward | Upload a file | +| DeleteObject | `DELETE /{bucket}/{key}` | Forward | Delete a file | +| ListBucket | `GET /{bucket}` | Response | List objects in a bucket (ListObjectsV2) | +| ListBuckets | `GET /` | Response | List all virtual buckets | +| CreateMultipartUpload | `POST /{bucket}/{key}?uploads` | NeedsBody | Initiate a multipart upload | +| UploadPart | `PUT /{bucket}/{key}?partNumber=N&uploadId=ID` | NeedsBody | Upload a part | +| CompleteMultipartUpload | `POST /{bucket}/{key}?uploadId=ID` | NeedsBody | Complete a multipart upload | +| AbortMultipartUpload | `DELETE /{bucket}/{key}?uploadId=ID` | NeedsBody | Abort a multipart upload | + +### Dispatch Types + +- **Forward** — A presigned URL is generated and returned to the runtime, which executes it with its native HTTP client. Bodies stream directly between client and backend without buffering. +- **Response** — The handler builds a complete response (XML for LIST, error responses) and returns it. No presigned URL involved. +- **NeedsBody** — The runtime collects the request body, then the handler signs and sends the request via raw HTTP (`backend.send_raw()`). Multipart only. + +## STS Operations + +| Operation | HTTP Method | Description | +|-----------|------------|-------------| +| AssumeRoleWithWebIdentity | `POST /?Action=AssumeRoleWithWebIdentity&...` | Exchange OIDC JWT for temporary credentials | + +## OIDC Discovery Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/.well-known/openid-configuration` | GET | OpenID Connect discovery document | +| `/.well-known/jwks.json` | GET | JSON Web Key Set (proxy's RSA public key) | + +These are served when `OIDC_PROVIDER_KEY` and `OIDC_PROVIDER_ISSUER` are configured. + +## Limitations + +- **LIST returns all results** — `object_store::list_with_delimiter()` fetches all pages internally. `IsTruncated` is always `false`. Continuation tokens and max-keys are not supported. +- **Multipart is S3 only** — Multipart operations use raw HTTP with `S3RequestSigner` and are gated to `backend_type = "s3"`. Non-S3 backends should use single PUT requests. +- **DeleteObject does not return confirmation** — The proxy forwards the DELETE and returns the backend's response status. From 306f284f6ade3744019c142c5375b23535bd8253 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sat, 28 Feb 2026 22:18:27 -0500 Subject: [PATCH 65/82] chore: rm CLI CLI now lives at https://github.com/source-cooperative/source-coop-cli --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index c9a74a0..6653ed4 100644 --- a/Makefile +++ b/Makefile @@ -25,12 +25,6 @@ run-server: run-workers: npx wrangler dev --cwd crates/runtimes/cf-workers -build-cli: - cargo build -p source-coop-cli - -build-cli-staging: - cargo build -p source-coop-cli --no-default-features --features staging - ci-fast: fmt clippy check-wasm ci: ci-fast test From 37b10ab8e617f3db643c8397135c2dfe27f48c0d Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 1 Mar 2026 14:43:13 -0800 Subject: [PATCH 66/82] ci: tmp disable versioning and deployment tooling --- .github/workflows/please-release.yaml | 7 ++++--- .github/workflows/staging-deploy.yaml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/please-release.yaml b/.github/workflows/please-release.yaml index 162efa9..72414f3 100644 --- a/.github/workflows/please-release.yaml +++ b/.github/workflows/please-release.yaml @@ -1,8 +1,9 @@ name: Run release-please on: - push: - branches: - - main + workflow_dispatch: + # push: + # branches: + # - main permissions: contents: write diff --git a/.github/workflows/staging-deploy.yaml b/.github/workflows/staging-deploy.yaml index 92b32f3..a023002 100644 --- a/.github/workflows/staging-deploy.yaml +++ b/.github/workflows/staging-deploy.yaml @@ -1,9 +1,9 @@ name: Deploy to Staging on: - push: - branches: - - main + # push: + # branches: + # - main workflow_dispatch: permissions: From 57a72f98030942c01a42ad6de27d5dbda9264b09 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 1 Mar 2026 14:52:43 -0800 Subject: [PATCH 67/82] ci(docs): appropriately set base path for ghpages --- .github/workflows/docs.yaml | 5 +++++ docs/.vitepress/config.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 20c2462..519f403 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -35,6 +35,9 @@ jobs: cache: pnpm cache-dependency-path: docs/pnpm-lock.yaml + - uses: actions/configure-pages@v5 + id: pages + - name: Install dependencies run: pnpm install --frozen-lockfile working-directory: docs @@ -42,6 +45,8 @@ jobs: - name: Build docs run: pnpm docs:build working-directory: docs + env: + VITEPRESS_BASE: ${{ steps.pages.outputs.base_path && format('{0}/', steps.pages.outputs.base_path) || '/' }} - uses: actions/upload-pages-artifact@v3 with: diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index bd1c24a..df22c5a 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -99,6 +99,7 @@ const adminSidebar = [ export default withMermaid( defineConfig({ + base: (process.env.VITEPRESS_BASE as `/${string}/` | undefined) ?? "/", title: "Source Data Proxy", description: "Multi-runtime S3 gateway proxy in Rust", From bbd7377c3e42c30837613281fc6d59ab0b9a95ad Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 1 Mar 2026 18:03:20 -0800 Subject: [PATCH 68/82] docs: fixup --- docs/.vitepress/theme/style.css | 51 +++++++++++++++++++++++++++++++-- docs/getting-started/index.md | 2 +- docs/index.md | 20 +++++++++---- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/docs/.vitepress/theme/style.css b/docs/.vitepress/theme/style.css index 4bd08e0..831ef2e 100644 --- a/docs/.vitepress/theme/style.css +++ b/docs/.vitepress/theme/style.css @@ -157,12 +157,59 @@ } /* Hero heading on homepage */ -.VPHero .name, -.VPHero .text { +.VPHero .name { font-family: var(--vp-font-family-heading) !important; letter-spacing: -0.03em; } +/* Hero text — subtitle size (matches default tagline sizing) */ +.VPHero .text { + font-family: var(--vp-font-family-base) !important; + font-size: 18px !important; + line-height: 28px !important; + font-weight: 500 !important; + letter-spacing: normal !important; + color: var(--vp-c-text-2); +} + +@media (min-width: 640px) { + .VPHero .text { + font-size: 20px !important; + line-height: 32px !important; + } +} + +@media (min-width: 960px) { + .VPHero .text { + font-size: 24px !important; + line-height: 36px !important; + } +} + +/* Hero tagline — uppercase label */ +.VPHero .tagline { + text-transform: uppercase; + font-size: 13px !important; + line-height: 20px !important; + font-weight: 500 !important; + letter-spacing: 0.05em !important; + color: var(--vp-c-text-3) !important; +} + +@media (min-width: 640px) { + .VPHero .tagline { + font-size: 14px !important; + line-height: 22px !important; + } +} + +@media (min-width: 960px) { + .VPHero .tagline { + font-size: 14px !important; + line-height: 22px !important; + } +} + /* Feature card titles */ .VPFeature .title { font-family: var(--vp-font-family-heading); diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 0d5ea6a..81de8ef 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -1,6 +1,6 @@ # Quick Start -This guide is for administrators setting up and running the Source Data Proxy. If you're a user looking to access data through an existing proxy, see the [User Guide](/guide/). +This guide is for administrators setting up and running their own Source Data Proxy. If you're a user looking to access data through an existing proxy, see the [User Guide](/guide/). The Source Data Proxy is a multi-runtime S3 gateway that proxies requests to backend object stores. This guide gets you running locally in minutes. diff --git a/docs/index.md b/docs/index.md index ed07d26..d98b188 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,8 +3,8 @@ layout: home hero: name: Source Data Proxy - text: Multi-runtime S3 gateway proxy - tagline: A Radiant Earth project. Stream S3-compatible requests to backend object stores with authentication, authorization, and zero-copy passthrough. + text: A multi-runtime S3 gateway proxy with authentication, authorization, and zero-copy passthrough. + tagline: 'Built for reuse. the Source Data Proxy is the backend API powering source.coop' actions: - theme: brand text: User Guide @@ -33,15 +33,15 @@ features: ## Why a Proxy? -Source Cooperative hosts open data from researchers and organizations around the world. That data lives on object storage — but object storage alone doesn't solve the problems that come with making data truly accessible. +[Source Cooperative](https://source.coop) hosts open data from researchers and organizations around the world. That data lives on object storage — but object storage alone doesn't solve the problems that come with making data truly accessible. ### One URL, any backend -A dataset might start on AWS S3, move to Cloudflare R2 to reduce egress costs, or get mirrored across providers for redundancy. The proxy gives every data product a stable URL (`data.source.coop/{account}/{dataset}/...`) regardless of where the bytes actually live. Backend migrations are invisible to consumers — no broken links, no client reconfiguration. +A dataset might start on AWS S3, move to Cloudflare R2 to reduce egress costs, or get mirrored across providers for redundancy. The proxy gives every data product a stable URL (`data.source.coop/{virtual-bucket}...`) regardless of where the bytes actually live. Backend migrations are invisible to consumers — no broken links, no client reconfiguration. ### Native S3 compatibility -Rather than inventing a new API, the proxy speaks the S3 protocol. This means the entire ecosystem of existing tools — `aws-cli`, `boto3`, DuckDB, the Rust `object_store` crate, GDAL, and hundreds of others — works out of the box. Users don't install a custom client or learn a new SDK. They just set an endpoint URL. +Rather than inventing a new API, the proxy speaks the S3 protocol. This means the entire ecosystem of existing tools — `aws-cli`, `boto3`, DuckDB, the Rust `object_store` crate, the Python `obstore` module, GDAL, and hundreds of others — work out of the box. Users don't install a custom client or learn a new SDK. They just set an endpoint URL. ### Metered access @@ -77,3 +77,13 @@ flowchart LR ``` The proxy sits between S3-compatible clients and backend object stores. It authenticates incoming requests, authorizes them against configured scopes, and forwards them to the appropriate backend using presigned URLs for zero-copy streaming. + +## Get Started + +### Access data on Source Cooperative + +The [User Guide](/guide/) covers how to interact with Source Cooperative's hosted data proxy at `data.source.coop` — browsing datasets, authenticating, and using standard S3 clients to read and write data. + +### Build your own data proxy + +The [Administration](/getting-started/) section covers how to deploy and configure the Source Data Proxy for your own project — setting up backends, defining buckets and roles, configuring authentication, and extending the proxy with custom resolvers and providers. From 5ec01ff39e230e6563047824f75f96ed8fad3730 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 10:23:57 -0800 Subject: [PATCH 69/82] chore(getting-started): use admonition for tip From 8ea93f84d138e2dd224cbca009f7d833177e9435 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 11:58:34 -0800 Subject: [PATCH 70/82] docs: utilize admonitions --- docs/architecture/crate-layout.md | 5 ++--- docs/architecture/request-lifecycle.md | 10 ++++------ docs/auth/backend-auth.md | 20 ++++++++------------ docs/auth/proxy-auth.md | 3 ++- docs/auth/sealed-tokens.md | 5 ++--- docs/configuration/buckets.md | 10 ++++------ docs/configuration/credentials.md | 3 ++- docs/configuration/providers/dynamodb.md | 5 ++--- docs/configuration/providers/postgres.md | 5 ++--- docs/configuration/roles.md | 3 ++- docs/deployment/cloudflare-workers.md | 12 ++++++------ docs/getting-started/index.md | 3 ++- docs/guide/client-usage.md | 3 ++- docs/guide/endpoints.md | 5 ++--- docs/reference/errors.md | 5 ++--- docs/reference/operations.md | 7 ++++--- 16 files changed, 48 insertions(+), 56 deletions(-) diff --git a/docs/architecture/crate-layout.md b/docs/architecture/crate-layout.md index 3870f84..c1c38ec 100644 --- a/docs/architecture/crate-layout.md +++ b/docs/architecture/crate-layout.md @@ -69,9 +69,8 @@ The Cloudflare Workers WASM runtime: - JS `ReadableStream` passthrough for zero-copy streaming - Config loading from env vars (`PROXY_CONFIG`) -::: warning -This crate is excluded from the workspace `default-members` because WASM types are `!Send` and won't compile on native targets. Always build with `--target wasm32-unknown-unknown`. -::: +> [!WARNING] +> This crate is excluded from the workspace `default-members` because WASM types are `!Send` and won't compile on native targets. Always build with `--target wasm32-unknown-unknown`. ### `source-coop` (lib) diff --git a/docs/architecture/request-lifecycle.md b/docs/architecture/request-lifecycle.md index 05556f8..4bea162 100644 --- a/docs/architecture/request-lifecycle.md +++ b/docs/architecture/request-lifecycle.md @@ -72,9 +72,8 @@ Used for: **LIST, errors, synthetic responses** For LIST operations, the handler calls `object_store::list_with_delimiter()` via the backend's store, builds S3 `ListObjectsV2` XML from the results, and returns it as a complete response. If a `ListRewrite` is configured, key prefixes are transformed in the XML. -::: info -LIST returns all results in a single response. `IsTruncated` is always `false`. The proxy does not support S3-style pagination with continuation tokens. -::: +> [!NOTE] +> LIST returns all results in a single response. `IsTruncated` is always `false`. The proxy does not support S3-style pagination with continuation tokens. ### `NeedsBody(PendingRequest)` @@ -82,9 +81,8 @@ Used for: **CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMul Multipart operations need the request body (e.g., the XML body for `CompleteMultipartUpload`). The runtime materializes the body, then calls `handler.handle_with_body()`, which signs the request using `S3RequestSigner` and sends it via `backend.send_raw()`. -::: warning -Multipart uploads are only supported for `backend_type = "s3"`. Non-S3 backends should use single PUT requests (object_store handles chunking internally). -::: +> [!WARNING] +> Multipart uploads are only supported for `backend_type = "s3"`. Non-S3 backends should use single PUT requests (object_store handles chunking internally). ## Response Header Forwarding diff --git a/docs/auth/backend-auth.md b/docs/auth/backend-auth.md index d6e68ce..1ef0dbe 100644 --- a/docs/auth/backend-auth.md +++ b/docs/auth/backend-auth.md @@ -113,9 +113,8 @@ When OIDC provider keys are configured, the proxy serves two well-known endpoint } ``` -::: warning -These endpoints must be publicly accessible. Cloud providers fetch them at JWT validation time to verify signatures. If they are behind a firewall or VPN, credential exchange will fail. -::: +> [!WARNING] +> These endpoints must be publicly accessible. Cloud providers fetch them at JWT validation time to verify signatures. If they are behind a firewall or VPN, credential exchange will fail. ### The Exchange Flow in Detail @@ -148,9 +147,8 @@ On subsequent requests, cached credentials are reused until they expire. --thumbprint-list ``` - ::: tip - To get the thumbprint, fetch the TLS certificate chain from your proxy's domain. AWS uses this to verify the HTTPS connection to the JWKS endpoint. - ::: + > [!TIP] + > To get the thumbprint, fetch the TLS certificate chain from your proxy's domain. AWS uses this to verify the HTTPS connection to the JWKS endpoint. 2. **Create an IAM Role** with a trust policy that allows the proxy to assume it: ```json @@ -210,9 +208,8 @@ On subsequent requests, cached credentials are reused until they expire. ### Azure Blob Storage -::: info Planned -Azure OIDC backend auth is planned but not yet implemented. The proxy currently supports Azure with static credentials only. -::: +> [!NOTE] +> **Planned** — Azure OIDC backend auth is planned but not yet implemented. The proxy currently supports Azure with static credentials only. **Planned setup:** @@ -223,9 +220,8 @@ Azure OIDC backend auth is planned but not yet implemented. The proxy currently ### Google Cloud Storage -::: info Planned -GCS OIDC backend auth is planned but not yet implemented. The proxy currently supports GCS with static credentials only. -::: +> [!NOTE] +> **Planned** — GCS OIDC backend auth is planned but not yet implemented. The proxy currently supports GCS with static credentials only. **Planned setup:** diff --git a/docs/auth/proxy-auth.md b/docs/auth/proxy-auth.md index 85c9ca1..d99baf6 100644 --- a/docs/auth/proxy-auth.md +++ b/docs/auth/proxy-auth.md @@ -23,7 +23,8 @@ backend_type = "s3" anonymous_access = true ``` -Anonymous access only allows `GetObject`, `HeadObject`, and `ListBucket`. Write operations always require authentication. +> [!NOTE] +> Anonymous access only allows `GetObject`, `HeadObject`, and `ListBucket`. Write operations always require authentication. ## Long-Lived Access Keys diff --git a/docs/auth/sealed-tokens.md b/docs/auth/sealed-tokens.md index 2b35ba7..192d99d 100644 --- a/docs/auth/sealed-tokens.md +++ b/docs/auth/sealed-tokens.md @@ -47,9 +47,8 @@ export SESSION_TOKEN_KEY="" This key must be the same across all instances of the proxy. If you rotate the key, all existing session tokens become invalid — clients will need to re-authenticate. -::: warning -`SESSION_TOKEN_KEY` is required for the Cloudflare Workers runtime. Without it, temporary credentials from STS cannot be verified on subsequent requests. -::: +> [!WARNING] +> `SESSION_TOKEN_KEY` is required for the Cloudflare Workers runtime. Without it, temporary credentials from STS cannot be verified on subsequent requests. ## Scope Behavior diff --git a/docs/configuration/buckets.md b/docs/configuration/buckets.md index 86edd21..f3c0a1c 100644 --- a/docs/configuration/buckets.md +++ b/docs/configuration/buckets.md @@ -55,9 +55,8 @@ secret_access_key = "..." ### Azure Blob Storage -::: info -Requires the `azure` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. -::: +> [!NOTE] +> Requires the `azure` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. ```toml [buckets.backend_options] @@ -75,9 +74,8 @@ access_key = "..." ### Google Cloud Storage -::: info -Requires the `gcp` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. -::: +> [!NOTE] +> Requires the `gcp` feature flag on `source-coop-core`. Enabled by default in the server runtime, not available in CF Workers. ```toml [buckets.backend_options] diff --git a/docs/configuration/credentials.md b/docs/configuration/credentials.md index f9cd425..15e17b0 100644 --- a/docs/configuration/credentials.md +++ b/docs/configuration/credentials.md @@ -48,7 +48,8 @@ Long-lived credentials are appropriate for: - **Development and testing** environments - **Environments without an OIDC provider** -For CI/CD workflows and user-facing applications, prefer [OIDC/STS temporary credentials](/auth/proxy-auth#oidcsts-temporary-credentials) for better security (automatic expiration, no stored secrets). +> [!TIP] +> For CI/CD workflows and user-facing applications, prefer [OIDC/STS temporary credentials](/auth/proxy-auth#oidcsts-temporary-credentials) — they expire automatically and avoid storing secrets in config. ## Disabling Credentials diff --git a/docs/configuration/providers/dynamodb.md b/docs/configuration/providers/dynamodb.md index 59df1d0..28682a6 100644 --- a/docs/configuration/providers/dynamodb.md +++ b/docs/configuration/providers/dynamodb.md @@ -28,6 +28,5 @@ The provider uses a single-table design with partition key (`PK`) and sort key ( - Serverless deployments where a database server isn't practical - High-availability requirements (DynamoDB's built-in replication) -::: tip -Wrap the DynamoDB provider with [CachedProvider](./cached) to reduce read costs and latency. -::: +> [!TIP] +> Wrap the DynamoDB provider with [CachedProvider](./cached) to reduce read costs and latency. diff --git a/docs/configuration/providers/postgres.md b/docs/configuration/providers/postgres.md index f7516de..73e924b 100644 --- a/docs/configuration/providers/postgres.md +++ b/docs/configuration/providers/postgres.md @@ -23,6 +23,5 @@ let provider = PostgresProvider::new(pool); - Relational data management preferences - Complex queries or joins with other application data -::: tip -Wrap the PostgreSQL provider with [CachedProvider](./cached) to reduce query load and latency. -::: +> [!TIP] +> Wrap the PostgreSQL provider with [CachedProvider](./cached) to reduce query load and latency. diff --git a/docs/configuration/roles.md b/docs/configuration/roles.md index 4f5454d..6bb33d9 100644 --- a/docs/configuration/roles.md +++ b/docs/configuration/roles.md @@ -104,7 +104,8 @@ Prefix matching follows these rules: - If the prefix ends with `/` or is empty: the key must start with the prefix - Otherwise: the key must equal the prefix exactly, or start with the prefix followed by `/` -This prevents a prefix like `data` from accidentally matching `data-private/secret.txt`. The prefix `data/` would only match keys under the `data/` directory. +> [!IMPORTANT] +> A prefix without a trailing `/` must match exactly or be followed by `/`. This prevents `data` from matching `data-private/secret.txt`. Use `data/` to restrict to that directory. ## Template Variables diff --git a/docs/deployment/cloudflare-workers.md b/docs/deployment/cloudflare-workers.md index 8499468..24ed80d 100644 --- a/docs/deployment/cloudflare-workers.md +++ b/docs/deployment/cloudflare-workers.md @@ -4,9 +4,10 @@ The CF Workers runtime deploys the proxy to Cloudflare's edge network. It compil ## Limitations -- **S3 backends only** — Azure and GCS are not supported on WASM -- **Static or API config only** — DynamoDB and Postgres providers require Tokio, which is unavailable -- **`SESSION_TOKEN_KEY` required** — Workers are stateless, so sealed tokens are the only way to persist temporary credentials +> [!WARNING] +> - **S3 backends only** — Azure and GCS are not supported on WASM +> - **Static or API config only** — DynamoDB and Postgres providers require Tokio, which is unavailable +> - **`SESSION_TOKEN_KEY` required** — Workers are stateless, so sealed tokens are the only way to persist temporary credentials ## Configuration @@ -65,9 +66,8 @@ cd crates/runtimes/cf-workers npx wrangler build ``` -::: warning -Always use `--target wasm32-unknown-unknown` when checking or building the CF Workers crate. It is excluded from the workspace `default-members` because WASM types won't compile on native targets. -::: +> [!WARNING] +> Always use `--target wasm32-unknown-unknown` when checking or building the CF Workers crate. It is excluded from the workspace `default-members` because WASM types won't compile on native targets. ## Development diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 81de8ef..08be9bf 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -1,6 +1,7 @@ # Quick Start -This guide is for administrators setting up and running their own Source Data Proxy. If you're a user looking to access data through an existing proxy, see the [User Guide](/guide/). +> [!NOTE] +> This guide is for administrators setting up and running their own Source Data Proxy. If you're a user looking to access data through an existing proxy, see the [User Guide](/guide/). The Source Data Proxy is a multi-runtime S3 gateway that proxies requests to backend object stores. This guide gets you running locally in minutes. diff --git a/docs/guide/client-usage.md b/docs/guide/client-usage.md index 638d834..1bfa89c 100644 --- a/docs/guide/client-usage.md +++ b/docs/guide/client-usage.md @@ -86,7 +86,8 @@ curl https://data.source.coop/public-data/hello.txt curl -I https://data.source.coop/public-data/hello.txt ``` -For authenticated requests, use aws-cli or an SDK that handles SigV4 signing. +> [!NOTE] +> Authenticated requests require SigV4 signing. Use aws-cli or an SDK rather than raw curl. ## Request Styles diff --git a/docs/guide/endpoints.md b/docs/guide/endpoints.md index 1d38c1a..602085c 100644 --- a/docs/guide/endpoints.md +++ b/docs/guide/endpoints.md @@ -97,6 +97,5 @@ flowchart TD | EC2 in us-east-1, data in us-west-2 | `data.source.coop` | Cross-region: Cloudflare backbone is faster than cross-region AWS traffic | | SageMaker in us-west-2 | `us-west-2.data.source.coop` | Same-region: zero egress for training data | -::: tip -All endpoints support the same authentication methods and S3 operations. Your credentials work across any endpoint — only the `endpoint_url` changes. -::: +> [!TIP] +> All endpoints support the same authentication methods and S3 operations. Your credentials work across any endpoint — only the `endpoint_url` changes. diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 569fe9e..79bc703 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -53,6 +53,5 @@ STS errors follow the AWS STS error format: ## Error Message Safety -For 5xx errors, the proxy returns generic messages to avoid leaking internal infrastructure details. The full error message is logged server-side but not exposed to clients. - -For 4xx errors, the proxy returns descriptive messages to help clients debug authentication and authorization issues. +> [!NOTE] +> For 5xx errors, the proxy returns generic messages to avoid leaking internal infrastructure details. The full error is logged server-side but not exposed to clients. For 4xx errors, descriptive messages are returned to help clients debug authentication and authorization issues. diff --git a/docs/reference/operations.md b/docs/reference/operations.md index 09388e8..4547d1d 100644 --- a/docs/reference/operations.md +++ b/docs/reference/operations.md @@ -38,6 +38,7 @@ These are served when `OIDC_PROVIDER_KEY` and `OIDC_PROVIDER_ISSUER` are configu ## Limitations -- **LIST returns all results** — `object_store::list_with_delimiter()` fetches all pages internally. `IsTruncated` is always `false`. Continuation tokens and max-keys are not supported. -- **Multipart is S3 only** — Multipart operations use raw HTTP with `S3RequestSigner` and are gated to `backend_type = "s3"`. Non-S3 backends should use single PUT requests. -- **DeleteObject does not return confirmation** — The proxy forwards the DELETE and returns the backend's response status. +> [!WARNING] +> - **LIST returns all results** — `object_store::list_with_delimiter()` fetches all pages internally. `IsTruncated` is always `false`. Continuation tokens and max-keys are not supported. +> - **Multipart is S3 only** — Multipart operations use raw HTTP with `S3RequestSigner` and are gated to `backend_type = "s3"`. Non-S3 backends should use single PUT requests. +> - **DeleteObject does not return confirmation** — The proxy forwards the DELETE and returns the backend's response status. From 82db817d85527dfbb097b95229c8df5f2dc4eb46 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 12:10:24 -0800 Subject: [PATCH 71/82] docs: customize tip and note colors --- docs/.vitepress/theme/style.css | 38 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/.vitepress/theme/style.css b/docs/.vitepress/theme/style.css index 831ef2e..a74bf41 100644 --- a/docs/.vitepress/theme/style.css +++ b/docs/.vitepress/theme/style.css @@ -51,11 +51,22 @@ --vp-c-gutter: rgba(44, 50, 51, 0.06); --vp-c-border: rgba(44, 50, 51, 0.15); - /* Tip callout — inherit brand */ - --vp-c-tip-1: var(--vp-c-brand-1); - --vp-c-tip-2: var(--vp-c-brand-2); - --vp-c-tip-3: var(--vp-c-brand-3); - --vp-c-tip-soft: var(--vp-c-brand-soft); + /* Note callout — blue */ + --vp-c-note-1: #3b82f6; + --vp-c-note-2: #2563eb; + --vp-c-note-3: #1d4ed8; + --vp-custom-block-note-border: rgba(59, 130, 246, 0.3); + --vp-custom-block-note-bg: rgba(59, 130, 246, 0.08); + --vp-custom-block-note-code-bg: rgba(59, 130, 246, 0.12); + + /* Tip callout — green */ + --vp-c-tip-1: #16a34a; + --vp-c-tip-2: #15803d; + --vp-c-tip-3: #166534; + --vp-c-tip-soft: rgba(22, 163, 74, 0.08); + --vp-custom-block-tip-border: rgba(22, 163, 74, 0.3); + --vp-custom-block-tip-bg: rgba(22, 163, 74, 0.08); + --vp-custom-block-tip-code-bg: rgba(22, 163, 74, 0.12); /* Navbar / sidebar */ --vp-nav-bg-color: #ffffff; @@ -77,6 +88,23 @@ --vp-c-brand-3: #b29e99; --vp-c-brand-soft: rgba(239, 235, 234, 0.1); + /* Note callout — blue */ + --vp-c-note-1: #60a5fa; + --vp-c-note-2: #93bbfd; + --vp-c-note-3: #3b82f6; + --vp-custom-block-note-border: rgba(96, 165, 250, 0.3); + --vp-custom-block-note-bg: rgba(96, 165, 250, 0.1); + --vp-custom-block-note-code-bg: rgba(96, 165, 250, 0.14); + + /* Tip callout — green */ + --vp-c-tip-1: #4ade80; + --vp-c-tip-2: #86efac; + --vp-c-tip-3: #22c55e; + --vp-c-tip-soft: rgba(74, 222, 128, 0.1); + --vp-custom-block-tip-border: rgba(74, 222, 128, 0.3); + --vp-custom-block-tip-bg: rgba(74, 222, 128, 0.1); + --vp-custom-block-tip-code-bg: rgba(74, 222, 128, 0.14); + /* Page background */ --vp-c-bg: #2c3233; --vp-c-bg-alt: #242a2b; From 50642b8fcbbee3e99d6e194558bde15e98866f94 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 14:58:54 -0800 Subject: [PATCH 72/82] docs: update to adapt to cli changes --- docs/architecture/crate-layout.md | 2 +- docs/guide/authentication.md | 48 +++++++++++++++---------------- docs/guide/client-usage.md | 2 +- docs/guide/endpoints.md | 4 +-- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/docs/architecture/crate-layout.md b/docs/architecture/crate-layout.md index c1c38ec..8b4aa69 100644 --- a/docs/architecture/crate-layout.md +++ b/docs/architecture/crate-layout.md @@ -83,7 +83,7 @@ Source Cooperative-specific resolver and API client: Command-line tool for OIDC authentication: - Browser-based OAuth2 Authorization Code + PKCE flow - `credential_process` integration with AWS SDKs -- Credential caching in `~/.source-coop/credentials/` +- Credential caching in OS keyring (with file fallback) ## Dependency Flow diff --git a/docs/guide/authentication.md b/docs/guide/authentication.md index 82ab47c..f1e31c4 100644 --- a/docs/guide/authentication.md +++ b/docs/guide/authentication.md @@ -38,7 +38,12 @@ The `source-coop` CLI handles the OIDC flow for you. It opens your browser, auth **Install the CLI:** ```bash -cargo install --path crates/cli +# macOS / Linux +curl --proto '=https' --tlsv1.2 -LsSf \ + https://github.com/source-cooperative/source-coop-cli/releases/latest/download/source-coop-cli-installer.sh | sh + +# Or from source +cargo install --git https://github.com/source-cooperative/source-coop-cli ``` **Log in:** @@ -47,7 +52,7 @@ cargo install --path crates/cli source-coop login ``` -This opens your browser to authenticate. Once complete, credentials are cached locally. +This opens your browser to authenticate. Once complete, credentials are cached in your OS keyring. ### AWS Profile Integration @@ -55,18 +60,10 @@ Set up an AWS profile to use the proxy seamlessly with standard AWS tools: ```ini [profile source-coop] -credential_process = source-coop credential-process +credential_process = source-coop creds endpoint_url = https://data.source.coop ``` -For local development, override the proxy URL: - -```ini -[profile sc-local] -credential_process = source-coop credential-process --proxy-url http://localhost:8787 -endpoint_url = http://localhost:8787 -``` - Then use AWS tools normally — credentials are obtained and refreshed automatically: ```bash @@ -85,33 +82,34 @@ source-coop login --role-arn admin-role ```ini [profile sc-reader] -credential_process = source-coop credential-process --role-arn reader-role +credential_process = source-coop creds --role-arn reader-role endpoint_url = https://data.source.coop [profile sc-admin] -credential_process = source-coop credential-process --role-arn admin-role +credential_process = source-coop creds --role-arn admin-role endpoint_url = https://data.source.coop ``` ### CLI Options -| Flag | Env Var | Default | Description | -|------|---------|---------|-------------| -| `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | -| `--client-id` | `SOURCE_OIDC_CLIENT_ID` | (built-in) | OAuth2 client ID | -| `--proxy-url` | `SOURCE_PROXY_URL` | `https://data.source.coop` | Proxy URL for STS | -| `--role-arn` | `SOURCE_ROLE_ARN` | `source-coop-user` | Role ARN to assume | -| `--format` | | `credential-process` | Output: `credential-process` or `env` | -| `--duration` | | (role default) | Session duration in seconds | -| `--scope` | | `openid` | OAuth2 scopes | -| `--port` | | `0` (random) | Local callback server port | +| Flag | Env Var | Default | Description | +| ------------- | ----------------------- | -------------------------- | ------------------------------------- | +| `--issuer` | `SOURCE_OIDC_ISSUER` | `https://auth.source.coop` | OIDC issuer URL | +| `--client-id` | `SOURCE_OIDC_CLIENT_ID` | (built-in) | OAuth2 client ID | +| `--proxy-url` | `SOURCE_PROXY_URL` | `https://data.source.coop` | Proxy URL for STS | +| `--role-arn` | `SOURCE_ROLE_ARN` | `source-coop-user` | Role ARN to assume | +| `--format` | | `credential-process` | Output: `credential-process` or `env` | +| `--duration` | | (role default) | Session duration in seconds | +| `--scope` | | `openid` | OAuth2 scopes | +| `--port` | | `0` (random) | Local callback server port | +| `--no-cache` | | | Skip caching credentials | ### Direct STS Exchange -For CI/CD pipelines and scripts, the CLI can output credentials as environment variables using `--format env`: +After logging in, you can export cached credentials as environment variables: ```bash -eval $(source-coop login --format env) +eval $(source-coop creds --format env) # Credentials are now exported — use any S3 client aws s3 cp ./data.csv s3://deploy-bundles/data.csv \ diff --git a/docs/guide/client-usage.md b/docs/guide/client-usage.md index 1bfa89c..0a84152 100644 --- a/docs/guide/client-usage.md +++ b/docs/guide/client-usage.md @@ -24,7 +24,7 @@ Add a profile to `~/.aws/config` to avoid specifying the endpoint every time: ```ini [profile source-coop] -credential_process = source-coop credential-process +credential_process = source-coop creds endpoint_url = https://data.source.coop ``` diff --git a/docs/guide/endpoints.md b/docs/guide/endpoints.md index 602085c..f5eddeb 100644 --- a/docs/guide/endpoints.md +++ b/docs/guide/endpoints.md @@ -60,12 +60,12 @@ Set up profiles for both global and regional access: ```ini # For general use (laptop, CI/CD outside AWS) [profile source-coop] -credential_process = source-coop credential-process +credential_process = source-coop creds endpoint_url = https://data.source.coop # For workloads in us-west-2 [profile source-coop-usw2] -credential_process = source-coop credential-process +credential_process = source-coop creds endpoint_url = https://us-west-2.data.source.coop ``` From 712f6a04072a1b7bd5c67090b761a9b2ad88583f Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:14:46 -0800 Subject: [PATCH 73/82] refactor: improve RNG for credentials --- Cargo.lock | 2 +- Cargo.toml | 1 + crates/libs/sts/Cargo.toml | 2 +- crates/libs/sts/src/sts.rs | 18 +++++++----------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b5b8ac..80c63fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2568,6 +2568,7 @@ dependencies = [ "base64", "chrono", "quick-xml 0.37.5", + "rand 0.8.5", "reqwest", "rsa", "serde", @@ -2577,7 +2578,6 @@ dependencies = [ "thiserror", "tracing", "url", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index df32245..da479bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ http = "1" chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1", features = ["v4", "js"] } base64 = "0.22" +rand = "0.8" hex = "0.4" url = "2" diff --git a/crates/libs/sts/Cargo.toml b/crates/libs/sts/Cargo.toml index 5096870..caa8958 100644 --- a/crates/libs/sts/Cargo.toml +++ b/crates/libs/sts/Cargo.toml @@ -12,8 +12,8 @@ thiserror.workspace = true serde.workspace = true serde_json.workspace = true chrono.workspace = true -uuid.workspace = true base64.workspace = true +rand.workspace = true rsa.workspace = true sha2.workspace = true reqwest.workspace = true diff --git a/crates/libs/sts/src/sts.rs b/crates/libs/sts/src/sts.rs index 94fed6f..698bc23 100644 --- a/crates/libs/sts/src/sts.rs +++ b/crates/libs/sts/src/sts.rs @@ -1,8 +1,8 @@ //! STS credential minting. use chrono::{Duration, Utc}; +use rand::RngCore; use source_coop_core::types::{AccessScope, RoleConfig, TemporaryCredentials}; -use uuid::Uuid; /// Resolve `{claim_name}` template variables in access scopes against JWT claims. /// @@ -75,8 +75,9 @@ pub fn mint_temporary_credentials( fn generate_random_id(len: usize) -> String { use base64::Engine; - let bytes: Vec = (0..len).map(|_| rand_byte()).collect(); - let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&bytes); + let mut bytes = vec![0u8; len]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes); // Take only alphanumeric chars to match AWS key format encoded .chars() @@ -87,14 +88,9 @@ fn generate_random_id(len: usize) -> String { fn generate_session_token() -> String { use base64::Engine; - let bytes: Vec = (0..32).map(|_| rand_byte()).collect(); - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&bytes) -} - -/// Simple random byte using UUID as entropy source (avoids extra deps). -fn rand_byte() -> u8 { - let id = Uuid::new_v4(); - id.as_bytes()[0] + let mut bytes = [0u8; 32]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes) } #[cfg(test)] From 987fbd24805cc826c92e37a936fd5605d8690cde Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:18:33 -0800 Subject: [PATCH 74/82] test(sts): improve testing for iam role subject glob --- crates/libs/sts/src/lib.rs | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/crates/libs/sts/src/lib.rs b/crates/libs/sts/src/lib.rs index 1db7954..e55c6fa 100644 --- a/crates/libs/sts/src/lib.rs +++ b/crates/libs/sts/src/lib.rs @@ -220,22 +220,97 @@ mod tests { #[test] fn test_subject_matching() { + // Trailing wildcard assert!(subject_matches( "repo:org/repo:ref:refs/heads/main", "repo:org/repo:*" )); + + // Match-all wildcard assert!(subject_matches("repo:org/repo:ref:refs/heads/main", "*")); + + // Exact match (no wildcards) assert!(subject_matches( "repo:org/repo:ref:refs/heads/main", "repo:org/repo:ref:refs/heads/main" )); + + // Wrong prefix assert!(!subject_matches( "repo:org/repo:ref:refs/heads/main", "repo:other/*" )); + + // Multiple wildcards assert!(subject_matches( "repo:org/repo:ref:refs/heads/main", "repo:org/*:ref:refs/heads/*" )); } + + #[test] + fn test_subject_matching_exact() { + assert!(subject_matches("abc", "abc")); + assert!(!subject_matches("abc", "abcd")); + assert!(!subject_matches("abcd", "abc")); + assert!(!subject_matches("", "abc")); + assert!(subject_matches("", "")); + } + + #[test] + fn test_subject_matching_leading_wildcard() { + assert!(subject_matches("anything", "*")); + assert!(subject_matches("", "*")); + assert!(subject_matches("foo", "*foo")); + assert!(subject_matches("xfoo", "*foo")); + assert!(!subject_matches("foox", "*foo")); + } + + #[test] + fn test_subject_matching_trailing_wildcard() { + assert!(subject_matches("foo", "foo*")); + assert!(subject_matches("foobar", "foo*")); + assert!(!subject_matches("xfoo", "foo*")); + } + + #[test] + fn test_subject_matching_middle_wildcard() { + assert!(subject_matches("foobar", "foo*bar")); + assert!(subject_matches("fooXbar", "foo*bar")); + assert!(subject_matches("fooXYZbar", "foo*bar")); + assert!(!subject_matches("fooXbaz", "foo*bar")); + assert!(!subject_matches("xfoobar", "foo*bar")); + } + + #[test] + fn test_subject_matching_multiple_wildcards() { + // Two wildcards with repeated literal + assert!(subject_matches("axbb", "a*b*b")); + assert!(!subject_matches("axb", "a*b*b")); + + // Wildcard must not overlap with suffix + assert!(!subject_matches("abc", "a*bc*c")); + assert!(subject_matches("abcc", "a*bc*c")); + + // Multiple wildcards requiring non-greedy left-to-right match + assert!(subject_matches("aab", "*a*ab")); + assert!(!subject_matches("xab", "*a*ab")); + + // Repeated pattern in subject + assert!(subject_matches("xababab", "*ab*ab")); + assert!(!subject_matches("xab", "*ab*ab")); + } + + #[test] + fn test_subject_matching_double_wildcard() { + assert!(subject_matches("anything", "**")); + assert!(subject_matches("", "**")); + } + + #[test] + fn test_subject_matching_empty_subject() { + assert!(subject_matches("", "*")); + assert!(!subject_matches("", "a")); + assert!(subject_matches("", "")); + } } From 40b5886f15f0ac29713462d4860563f29493b796 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:24:03 -0800 Subject: [PATCH 75/82] fix(sts): harden JWKS fetching with HTTPS enforcement and failure backoff Reject non-HTTPS OIDC issuer URLs per the OIDC spec to prevent MITM attacks. Cache failed JWKS fetches for 30s to avoid hammering broken endpoints on repeated STS requests. Co-Authored-By: Claude Opus 4.6 --- crates/libs/sts/src/jwks.rs | 55 +++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/libs/sts/src/jwks.rs b/crates/libs/sts/src/jwks.rs index 3419a77..84b70fe 100644 --- a/crates/libs/sts/src/jwks.rs +++ b/crates/libs/sts/src/jwks.rs @@ -32,12 +32,22 @@ pub struct JwkKey { } /// Fetch JWKS from an OIDC provider's well-known endpoint. +/// +/// Requires HTTPS issuer URLs per the OIDC specification. HTTP URLs are +/// rejected to prevent MITM attacks on JWKS discovery. pub async fn fetch_jwks( client: &reqwest::Client, issuer: &str, ) -> Result { let issuer = issuer.trim_end_matches('/'); + if !issuer.starts_with("https://") { + return Err(ProxyError::ConfigError(format!( + "OIDC issuer must use HTTPS: {}", + issuer + ))); + } + // First, try the .well-known/openid-configuration endpoint let config_url = format!("{}/.well-known/openid-configuration", issuer); @@ -200,12 +210,17 @@ pub fn verify_token( /// a network round-trip to the provider on every token validation and prevents /// DoS via repeated validation attempts. /// +/// Failed fetches are cached with a shorter TTL (`failure_ttl`) to avoid +/// hammering broken endpoints while still retrying periodically. +/// /// Uses `DateTime` instead of `std::time::Instant` for WASM compatibility /// (`Instant` panics on `wasm32-unknown-unknown`). pub struct JwksCache { client: reqwest::Client, ttl: Duration, + failure_ttl: Duration, entries: Mutex, JwksResponse)>>, + failures: Mutex>>, } impl JwksCache { @@ -214,7 +229,9 @@ impl JwksCache { Self { client, ttl, + failure_ttl: Duration::from_secs(30), entries: Mutex::new(HashMap::new()), + failures: Mutex::new(HashMap::new()), } } @@ -225,13 +242,39 @@ impl JwksCache { return Ok(cached); } - // Cache miss — fetch from the network - let jwks = fetch_jwks(&self.client, issuer).await?; - - let mut entries = self.entries.lock().unwrap(); - entries.insert(issuer.to_string(), (Utc::now(), jwks.clone())); + // Check if we recently failed for this issuer + { + let failures = self.failures.lock().unwrap(); + if let Some(failed_at) = failures.get(issuer) { + let elapsed = Utc::now().signed_duration_since(*failed_at).num_seconds(); + if elapsed >= 0 && (elapsed as u64) < self.failure_ttl.as_secs() { + return Err(ProxyError::InvalidOidcToken(format!( + "JWKS fetch for '{}' recently failed, retrying after backoff", + issuer + ))); + } + } + } - Ok(jwks) + // Cache miss — fetch from the network + match fetch_jwks(&self.client, issuer).await { + Ok(jwks) => { + let mut entries = self.entries.lock().unwrap(); + entries.insert(issuer.to_string(), (Utc::now(), jwks.clone())); + // Clear any failure state on success + drop(entries); + self.failures.lock().unwrap().remove(issuer); + Ok(jwks) + } + Err(e) => { + tracing::warn!(issuer = %issuer, error = %e, "JWKS fetch failed, backing off"); + self.failures + .lock() + .unwrap() + .insert(issuer.to_string(), Utc::now()); + Err(e) + } + } } fn get_cached(&self, issuer: &str) -> Option { From f8e753492be09c047a54fbe0e59c20f27aa4d08a Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:24:27 -0800 Subject: [PATCH 76/82] fix(auth): reduce sensitive data in SigV4 mismatch logs Move canonical request details to debug level and stop logging expected/provided signatures entirely. Add access key and token context to sealed token unsealing failures for easier debugging. Co-Authored-By: Claude Opus 4.6 --- crates/libs/core/src/auth.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/libs/core/src/auth.rs b/crates/libs/core/src/auth.rs index 0d4b647..2dbfbdf 100644 --- a/crates/libs/core/src/auth.rs +++ b/crates/libs/core/src/auth.rs @@ -142,11 +142,14 @@ pub fn verify_sigv4_signature( if !matched { tracing::warn!( + access_key_id = %auth.access_key_id, + region = %auth.region, + "SigV4 signature mismatch" + ); + tracing::debug!( canonical_request = %canonical_request, string_to_sign = %string_to_sign, - expected_signature = %expected_signature, - provided_signature = %auth.signature, - "SigV4 signature mismatch — compare canonical_request with client-side (aws --debug)" + "SigV4 signature mismatch details — compare canonical_request with client-side (aws --debug)" ); } @@ -248,7 +251,11 @@ pub async fn resolve_identity( return Ok(ResolvedIdentity::Temporary { credentials: creds }); } None => { - tracing::warn!("session token could not be unsealed (decryption failed)"); + tracing::warn!( + access_key_id = %sig.access_key_id, + token_len = session_token.len(), + "session token could not be unsealed — possible key mismatch, token corruption, or expired key rotation" + ); return Err(ProxyError::AccessDenied); } } From 149877470db61f138a4b46b717a360b430be7662 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:24:40 -0800 Subject: [PATCH 77/82] docs(sealed_token): document security properties Co-Authored-By: Claude Opus 4.6 --- crates/libs/core/src/sealed_token.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/libs/core/src/sealed_token.rs b/crates/libs/core/src/sealed_token.rs index c82b0f3..17fca08 100644 --- a/crates/libs/core/src/sealed_token.rs +++ b/crates/libs/core/src/sealed_token.rs @@ -5,6 +5,17 @@ //! no server-side storage lookup is needed. This is critical for stateless //! runtimes like Cloudflare Workers where in-memory state does not persist //! across invocations. +//! +//! ## Security properties +//! +//! - **Encryption**: AES-256-GCM provides authenticated encryption (confidentiality + integrity). +//! - **Nonce**: 12-byte random nonce per token via `OsRng` (96 bits, per GCM spec). +//! - **Token format**: `base64url(nonce[12] || ciphertext + GCM tag[16])`. +//! - **Expiration**: Enforced at unseal time — expired credentials return `Err`. +//! - **Scope binding**: Allowed scopes are sealed at mint time, so config changes only affect +//! newly minted credentials. +//! - **Key rotation**: Tokens sealed with an old key will fail to decrypt (`Ok(None)`), causing +//! the client to re-authenticate. No explicit revocation mechanism is needed. use crate::error::ProxyError; use crate::types::TemporaryCredentials; From 421ea255344c6eae38732c16e0a0b3280848d2a3 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:30:08 -0800 Subject: [PATCH 78/82] perf(proxy): parse LIST query string once instead of three times Extract prefix, delimiter, and pagination params (max-keys, continuation-token, start-after) in a single pass over the query string instead of calling url::form_urlencoded::parse() three separate times per LIST request. Co-Authored-By: Claude Opus 4.6 --- crates/libs/core/src/proxy.rs | 80 +++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index b46d68a..b44ee89 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -402,26 +402,13 @@ where ) -> Result { let store = self.backend.create_store(config)?; - // Extract prefix from query string - let client_prefix = raw_query - .and_then(|q| { - url::form_urlencoded::parse(q.as_bytes()) - .find(|(k, _)| k == "prefix") - .map(|(_, v)| v.to_string()) - }) - .unwrap_or_default(); - - // Extract delimiter from query string (default "/") - let delimiter = raw_query - .and_then(|q| { - url::form_urlencoded::parse(q.as_bytes()) - .find(|(k, _)| k == "delimiter") - .map(|(_, v)| v.to_string()) - }) - .unwrap_or_else(|| "/".to_string()); + // Parse all query parameters in a single pass + let list_params = parse_list_query_params(raw_query); + let client_prefix = &list_params.prefix; + let delimiter = &list_params.delimiter; // Build the full prefix including backend_prefix - let full_prefix = build_list_prefix(config, &client_prefix); + let full_prefix = build_list_prefix(config, client_prefix); tracing::debug!( full_prefix = %full_prefix, @@ -444,12 +431,12 @@ where let bucket_name = &config.name; let xml = build_list_xml( bucket_name, - &client_prefix, - &delimiter, + client_prefix, + delimiter, &list_result, config, list_rewrite, - raw_query, + &list_params.pagination, )?; let mut resp_headers = HeaderMap::new(); @@ -624,7 +611,7 @@ fn build_list_xml( list_result: &object_store::ListResult, config: &BucketConfig, list_rewrite: Option<&ListRewrite>, - raw_query: Option<&str>, + params: &pagination::PaginationParams, ) -> Result { let backend_prefix = config .backend_prefix @@ -666,8 +653,7 @@ fn build_list_xml( }) .collect(); - let params = pagination::parse_pagination_params(raw_query); - let page = paginate(contents, common_prefixes, ¶ms)?; + let page = paginate(contents, common_prefixes, params)?; Ok(ListBucketResult { xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", @@ -677,8 +663,8 @@ fn build_list_xml( max_keys: params.max_keys, is_truncated: page.is_truncated, key_count: page.contents.len() + page.common_prefixes.len(), - start_after: params.start_after, - continuation_token: params.continuation_token, + start_after: params.start_after.clone(), + continuation_token: params.continuation_token.clone(), next_continuation_token: page.next_continuation_token, contents: page.contents, common_prefixes: page.common_prefixes, @@ -716,6 +702,48 @@ fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite> key } +/// All query parameters needed for a LIST operation, parsed in a single pass. +struct ListQueryParams { + prefix: String, + delimiter: String, + pagination: pagination::PaginationParams, +} + +/// Parse prefix, delimiter, and pagination params from a LIST query string in one pass. +fn parse_list_query_params(raw_query: Option<&str>) -> ListQueryParams { + let mut prefix = None; + let mut delimiter = None; + let mut max_keys = None; + let mut continuation_token = None; + let mut start_after = None; + + if let Some(q) = raw_query { + for (k, v) in url::form_urlencoded::parse(q.as_bytes()) { + match k.as_ref() { + "prefix" => prefix = Some(v.into_owned()), + "delimiter" => delimiter = Some(v.into_owned()), + "max-keys" => max_keys = Some(v.into_owned()), + "continuation-token" => continuation_token = Some(v.into_owned()), + "start-after" => start_after = Some(v.into_owned()), + _ => {} + } + } + } + + ListQueryParams { + prefix: prefix.unwrap_or_default(), + delimiter: delimiter.unwrap_or_else(|| "/".to_string()), + pagination: pagination::PaginationParams { + max_keys: max_keys + .and_then(|v| v.parse().ok()) + .unwrap_or(1000) + .min(1000), + continuation_token, + start_after, + }, + } +} + /// Build the backend URL for an S3 operation. /// /// Used for multipart operations that go through raw signed HTTP. From 13c82d3d17744f612dfd61dddfd974d5a3aa2ec9 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:30:49 -0800 Subject: [PATCH 79/82] perf(proxy): reduce string allocations in LIST key rewriting Work with &str slices through prefix stripping operations and only allocate the final String once, instead of creating 2-3 intermediate String allocations per object in LIST responses. Co-Authored-By: Claude Opus 4.6 --- crates/libs/core/src/proxy.rs | 42 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index b44ee89..709d7d8 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -673,33 +673,39 @@ fn build_list_xml( } /// Apply strip/add prefix rewriting to a key or prefix value. +/// +/// Works with `&str` slices to avoid intermediate allocations — only allocates +/// the final `String` once. fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite>) -> String { - let mut key = raw.to_string(); - - // Strip the backend prefix - if !strip_prefix.is_empty() { - if let Some(stripped) = key.strip_prefix(strip_prefix) { - key = stripped.to_string(); - } - } + // Strip the backend prefix (borrow from `raw`, no allocation) + let key = if !strip_prefix.is_empty() { + raw.strip_prefix(strip_prefix).unwrap_or(raw) + } else { + raw + }; // Apply list_rewrite if present if let Some(rewrite) = list_rewrite { - if !rewrite.strip_prefix.is_empty() { - if let Some(stripped) = key.strip_prefix(&rewrite.strip_prefix) { - key = stripped.to_string(); - } - } + let key = if !rewrite.strip_prefix.is_empty() { + key.strip_prefix(rewrite.strip_prefix.as_str()) + .unwrap_or(key) + } else { + key + }; + if !rewrite.add_prefix.is_empty() { - if key.is_empty() || key.starts_with('/') { - key = format!("{}{}", rewrite.add_prefix, key); + // Must allocate for add_prefix — early return + return if key.is_empty() || key.starts_with('/') { + format!("{}{}", rewrite.add_prefix, key) } else { - key = format!("{}/{}", rewrite.add_prefix, key); - } + format!("{}/{}", rewrite.add_prefix, key) + }; } + + return key.to_string(); } - key + key.to_string() } /// All query parameters needed for a LIST operation, parsed in a single pass. From 898b4defb2dd4aacc61e69ee6dea95fd6e78c076 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:31:31 -0800 Subject: [PATCH 80/82] perf(proxy): pre-allocate in path building helpers Use String::with_capacity() in build_object_path and build_list_prefix to avoid reallocations. Skip allocation entirely when no backend_prefix is configured. Co-Authored-By: Claude Opus 4.6 --- crates/libs/core/src/proxy.rs | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index 709d7d8..f4e3860 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -577,30 +577,40 @@ fn sign_s3_request( /// Build an object_store Path from a bucket config and client-visible key. fn build_object_path(config: &BucketConfig, key: &str) -> object_store::path::Path { - let mut full_key = String::new(); - if let Some(prefix) = &config.backend_prefix { - let p = prefix.trim_end_matches('/'); - if !p.is_empty() { - full_key.push_str(p); - full_key.push('/'); + match &config.backend_prefix { + Some(prefix) => { + let p = prefix.trim_end_matches('/'); + if p.is_empty() { + object_store::path::Path::from(key) + } else { + let mut full_key = String::with_capacity(p.len() + 1 + key.len()); + full_key.push_str(p); + full_key.push('/'); + full_key.push_str(key); + object_store::path::Path::from(full_key) + } } + None => object_store::path::Path::from(key), } - full_key.push_str(key); - object_store::path::Path::from(full_key) } /// Build the full list prefix including backend_prefix. fn build_list_prefix(config: &BucketConfig, client_prefix: &str) -> String { - let mut full_prefix = String::new(); - if let Some(bp) = &config.backend_prefix { - let bp = bp.trim_end_matches('/'); - if !bp.is_empty() { - full_prefix.push_str(bp); - full_prefix.push('/'); + match &config.backend_prefix { + Some(prefix) => { + let bp = prefix.trim_end_matches('/'); + if bp.is_empty() { + client_prefix.to_string() + } else { + let mut full_prefix = String::with_capacity(bp.len() + 1 + client_prefix.len()); + full_prefix.push_str(bp); + full_prefix.push('/'); + full_prefix.push_str(client_prefix); + full_prefix + } } + None => client_prefix.to_string(), } - full_prefix.push_str(client_prefix); - full_prefix } /// Build S3 ListObjectsV2 XML from an object_store ListResult. From 4e141d07398787e044289d0f89494cdaecb4cfa4 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 2 Mar 2026 22:56:29 -0800 Subject: [PATCH 81/82] perf(proxy): use PaginatedListStore for backend-side LIST pagination Replace `list_with_delimiter()` (which loads all results into memory) with `PaginatedListStore::list_paginated()` from object_store 0.13.1. Pagination params (max-keys, continuation-token, start-after) are now pushed to the backend, fetching only one page per request. - Add `create_paginated_store()` to `ProxyBackend` trait - Add `build_paginated_list_store()` and `StoreBuilder::build_paginated()` - Update `handle_list()` to use `list_paginated()` with `PaginatedListOptions` - Pass through backend's opaque page tokens as continuation tokens - Both server and CF Workers runtimes implement the new trait method Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 17 ++-- crates/libs/core/src/backend.rs | 49 ++++++++-- crates/libs/core/src/proxy.rs | 115 +++++++++++++++-------- crates/runtimes/cf-workers/src/client.rs | 11 ++- crates/runtimes/server/src/client.rs | 13 ++- 5 files changed, 142 insertions(+), 63 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0567107..b8b75b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,13 +31,13 @@ cargo test - `Forward(ForwardRequest)` — presigned URL + headers for GET/HEAD/PUT/DELETE. The runtime executes the request with its native HTTP client, enabling zero-copy streaming. - `Response(ProxyResult)` — complete response for LIST, errors, synthetic responses. - `NeedsBody(PendingRequest)` — multipart operations that need the request body. The runtime materializes the body and calls `handle_with_body()`. -- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend`, `R: RequestResolver`, and `O: OidcBackendAuth` (defaults to `NoOidcAuth`). The backend trait has three methods: `create_store()` returns an `Arc` for LIST, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: - - **Server**: `ServerBackend` delegates to `build_object_store()` (with default connector) and `build_signer()`, and uses reqwest for raw HTTP + Forward execution. - - **CF Workers**: `WorkerBackend` delegates to `build_object_store()` (injecting `FetchConnector`) and `build_signer()`, and uses `web_sys::fetch` for raw HTTP + Forward execution. -- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_object_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_object_store()` only — `build_signer()` needs no connector since signing is pure computation. +- **ProxyBackend trait**: `ProxyHandler` is generic over `B: ProxyBackend`, `R: RequestResolver`, and `O: OidcBackendAuth` (defaults to `NoOidcAuth`). The backend trait has three methods: `create_paginated_store()` returns a `Box` for LIST with backend-side pagination, `create_signer()` returns an `Arc` for presigned URL generation, and `send_raw()` sends pre-signed HTTP requests for multipart operations. Each runtime provides its own implementation: + - **Server**: `ServerBackend` delegates to `build_paginated_list_store()` (with default connector) and `build_signer()`, and uses reqwest for raw HTTP + Forward execution. + - **CF Workers**: `WorkerBackend` delegates to `build_paginated_list_store()` (injecting `FetchConnector`) and `build_signer()`, and uses `web_sys::fetch` for raw HTTP + Forward execution. +- **Multi-provider support**: `BucketConfig` uses a `backend_type: String` discriminator (`"s3"`, `"az"`, `"gcs"`) and a `backend_options: HashMap` for provider-specific config. `build_paginated_list_store()` in `crates/libs/core/src/backend.rs` dispatches on `backend_type`, iterating `backend_options` calling `with_config()` on the appropriate builder. `build_signer()` dispatches similarly: for authenticated backends it uses `object_store`'s built-in signer (WASM-safe because `StaticCredentialProvider` bypasses `Instant::now()`); for anonymous backends (no credentials) it returns `UnsignedUrlSigner` which constructs plain URLs without auth parameters (avoiding the `InstanceCredentialProvider` → `Instant::now()` panic on WASM). Azure and GCS builders are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime enables neither (only S3 is supported on WASM). Runtimes inject their HTTP connector via a closure over `StoreBuilder` for `build_paginated_list_store()` only — `build_signer()` needs no connector since signing is pure computation. - **Operation dispatch** via presigned URLs and direct object_store: - **GET/HEAD/PUT/DELETE** → `create_signer()` generates a presigned URL, returned as `HandlerAction::Forward`. The runtime executes the URL with its native HTTP client, streaming request/response bodies directly without handler involvement. - - **LIST** → `create_store()` + `store.list_with_delimiter()`; builds S3 ListObjectsV2 XML from `ListResult`. `IsTruncated` is always `false`. + - **LIST** → `create_paginated_store()` + `store.list_paginated()` via `PaginatedListStore`; `max-keys`, `continuation-token`, and `start-after` are pushed to the backend, fetching only one page per request. Builds S3 ListObjectsV2 XML from the paginated `ListResult`. - **Multipart** (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload) → `NeedsBody` then raw signed HTTP via `backend.send_raw()` + `S3RequestSigner`. - **ProxyResponseBody**: A simple enum (`Bytes`, `Empty`) for non-streaming responses only. Streaming bodies bypass this type entirely via the `Forward` action — runtimes handle them natively. - **RequestResolver pattern**: The resolver decides what to do with each request (parse, auth, authorize, return proxy action or synthetic response). `DefaultResolver` handles standard S3 proxy behavior. Custom resolvers (e.g., `SourceCoopResolver` in cf-workers) implement product-specific namespace mapping and auth. Runtimes are thin adapters that pick a resolver and call `handler.resolve_request()`. @@ -47,14 +47,13 @@ cargo test - **cf-workers is excluded from `default-members`** in the root `Cargo.toml` because WASM types are `!Send` and will fail to compile on native targets. Always use `--target wasm32-unknown-unknown` when working with this crate. - **Config loading** (CF Workers): `PROXY_CONFIG` can be either a JSON string (via `wrangler secret`) or a JS object (via `[vars.PROXY_CONFIG]` table in `wrangler.toml`). Both formats are handled. - **Sealed session tokens**: When `SESSION_TOKEN_KEY` is configured, temporary credentials minted by STS are AES-256-GCM encrypted into the session token itself (`sealed_token.rs`). On subsequent requests, `resolve_identity()` decrypts the token to recover credentials — no server-side storage or config lookup needed. This is required for stateless runtimes (CF Workers). `TokenKey` wraps `Arc` (Clone + Send + Sync). Token format: `base64url(nonce[12] || ciphertext + tag)`. Scopes are sealed at mint time, so config changes to `allowed_scopes` only affect newly minted credentials. The `DefaultResolver` accepts an optional `TokenKey` as its third constructor argument; the STS handler requires it when processing STS requests. -- **List response construction**: LIST responses are built directly from `object_store::ListResult` as S3 XML. When a resolver returns a `ListRewrite`, prefix stripping/adding is applied to `ObjectMeta.location` and `common_prefixes` paths before XML generation. The `list_rewrite` module in `crates/libs/core/src/s3/list_rewrite.rs` is retained for backward compatibility. -- **OIDC backend auth**: The `OidcBackendAuth` trait (`crates/libs/core/src/oidc_backend.rs`) resolves backend credentials via OIDC token exchange. When a bucket's `backend_options` contains `auth_type=oidc`, the proxy mints a self-signed JWT and exchanges it for temporary cloud credentials before the request reaches `create_store()`/`create_signer()`. The resolved credentials are injected into a cloned `BucketConfig.backend_options` so the existing builder pipeline works unmodified. `AwsOidcBackendAuth` (in `crates/libs/oidc-provider/src/backend_auth.rs`) implements this for AWS via `AssumeRoleWithWebIdentity`. `MaybeOidcAuth` is an enum (`Enabled`/`Disabled`) used as the concrete `O` type by both runtimes. OIDC is configured via `OIDC_PROVIDER_KEY` (PEM secret) and `OIDC_PROVIDER_ISSUER` (URL). When configured, `/.well-known/openid-configuration` and `/.well-known/jwks.json` are served for cloud provider JWKS discovery. The `S3RequestSigner` includes `x-amz-security-token` for STS temporary credentials. Currently AWS/S3 only; Azure and GCP exchange flows are TODO. +- **List response construction**: LIST responses are built from `object_store::ListResult` (one page at a time via `PaginatedListStore`) as S3 XML. When a resolver returns a `ListRewrite`, prefix stripping/adding is applied to `ObjectMeta.location` and `common_prefixes` paths before XML generation. The `list_rewrite` module in `crates/libs/core/src/s3/list_rewrite.rs` is retained for backward compatibility. +- **OIDC backend auth**: The `OidcBackendAuth` trait (`crates/libs/core/src/oidc_backend.rs`) resolves backend credentials via OIDC token exchange. When a bucket's `backend_options` contains `auth_type=oidc`, the proxy mints a self-signed JWT and exchanges it for temporary cloud credentials before the request reaches `create_paginated_store()`/`create_signer()`. The resolved credentials are injected into a cloned `BucketConfig.backend_options` so the existing builder pipeline works unmodified. `AwsOidcBackendAuth` (in `crates/libs/oidc-provider/src/backend_auth.rs`) implements this for AWS via `AssumeRoleWithWebIdentity`. `MaybeOidcAuth` is an enum (`Enabled`/`Disabled`) used as the concrete `O` type by both runtimes. OIDC is configured via `OIDC_PROVIDER_KEY` (PEM secret) and `OIDC_PROVIDER_ISSUER` (URL). When configured, `/.well-known/openid-configuration` and `/.well-known/jwks.json` are served for cloud provider JWKS discovery. The `S3RequestSigner` includes `x-amz-security-token` for STS temporary credentials. Currently AWS/S3 only; Azure and GCP exchange flows are TODO. ## Known Limitations 1. **Multipart uses raw HTTP (S3 only)**: `object_store`'s `MultipartUpload` API doesn't expose upload IDs. Multipart operations use `S3RequestSigner` + raw HTTP. They are gated to `backend_type == "s3"` — non-S3 backends return an error for multipart requests and should use `PUT` (object_store handles chunking internally). -2. **LIST returns all results**: `object_store::list_with_delimiter()` fetches all pages internally. No S3-style pagination (continuation tokens, max-keys truncation). `IsTruncated` is always `false`. -3. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. +2. **Azure/GCS require feature flags**: `MicrosoftAzureBuilder` and `GoogleCloudStorageBuilder` are gated behind cargo features (`azure`, `gcp`) on `source-coop-core`. The server runtime enables both; the CF Workers runtime only supports S3. Requesting an unsupported backend_type returns a `ConfigError`. ## Style diff --git a/crates/libs/core/src/backend.rs b/crates/libs/core/src/backend.rs index b7e46d1..5f5a384 100644 --- a/crates/libs/core/src/backend.rs +++ b/crates/libs/core/src/backend.rs @@ -3,15 +3,15 @@ //! [`ProxyBackend`] is the main trait runtimes implement. It provides three //! capabilities: //! -//! 1. **`create_store()`** — build an `ObjectStore` for high-level operations -//! (LIST) routed through `object_store`. +//! 1. **`create_paginated_store()`** — build a `PaginatedListStore` for LIST +//! operations with backend-side pagination. //! 2. **`create_signer()`** — build a `Signer` for generating presigned URLs //! for GET, HEAD, PUT, DELETE operations. //! 3. **`send_raw()`** — send a pre-signed HTTP request for operations not //! covered by `ObjectStore` (multipart uploads). //! //! [`S3RequestSigner`] is retained for signing multipart requests. -//! [`build_object_store`] and [`build_signer`] dispatch on +//! [`build_paginated_list_store`] and [`build_signer`] dispatch on //! `BucketConfig::backend_type` to build the appropriate provider. //! [`build_signer`] uses `object_store`'s built-in signer for authenticated //! backends, and [`UnsignedUrlSigner`] for anonymous backends (avoiding @@ -23,6 +23,7 @@ use crate::types::{BackendType, BucketConfig}; use bytes::Bytes; use http::HeaderMap; use object_store::aws::AmazonS3Builder; +use object_store::list::PaginatedListStore; use object_store::signer::Signer; use object_store::ObjectStore; use std::future::Future; @@ -39,11 +40,15 @@ use object_store::gcp::GoogleCloudStorageBuilder; /// - Server runtime: uses `reqwest` for raw HTTP, default `object_store` HTTP connector /// - Worker runtime: uses `web_sys::fetch` for raw HTTP, custom `FetchConnector` for `object_store` pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { - /// Create an `ObjectStore` for the given bucket configuration. + /// Create a [`PaginatedListStore`] for the given bucket configuration. /// - /// Used for LIST operations where `object_store` handles pagination - /// and response parsing internally. - fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; + /// Used for LIST operations with backend-side pagination via + /// [`PaginatedListStore::list_paginated`], avoiding loading all results + /// into memory. + fn create_paginated_store( + &self, + config: &BucketConfig, + ) -> Result, ProxyError>; /// Create a `Signer` for generating presigned URLs. /// @@ -100,6 +105,23 @@ impl StoreBuilder { } } + /// Build a `PaginatedListStore` for backend-side paginated listing. + pub fn build_paginated(self) -> Result, ProxyError> { + match self { + StoreBuilder::S3(b) => Ok(Box::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build S3 paginated store: {}", e)) + })?)), + #[cfg(feature = "azure")] + StoreBuilder::Azure(b) => Ok(Box::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build Azure paginated store: {}", e)) + })?)), + #[cfg(feature = "gcp")] + StoreBuilder::Gcs(b) => Ok(Box::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build GCS paginated store: {}", e)) + })?)), + } + } + /// Build a `Signer` for presigned URL generation. pub fn build_signer(self) -> Result, ProxyError> { match self { @@ -183,6 +205,19 @@ where configure(create_builder(config)?).build() } +/// Build a [`PaginatedListStore`] from a [`BucketConfig`], dispatching on `backend_type`. +/// +/// Like [`build_object_store`], accepts a configure closure for HTTP connector injection. +pub fn build_paginated_list_store( + config: &BucketConfig, + configure: F, +) -> Result, ProxyError> +where + F: FnOnce(StoreBuilder) -> StoreBuilder, +{ + configure(create_builder(config)?).build_paginated() +} + /// Build a [`Signer`] from a [`BucketConfig`], dispatching on `backend_type`. /// /// For backends with credentials, uses `object_store`'s built-in signer diff --git a/crates/libs/core/src/proxy.rs b/crates/libs/core/src/proxy.rs index f4e3860..0abbb75 100644 --- a/crates/libs/core/src/proxy.rs +++ b/crates/libs/core/src/proxy.rs @@ -19,12 +19,12 @@ use crate::error::ProxyError; use crate::oidc_backend::{NoOidcAuth, OidcBackendAuth}; use crate::resolver::{ListRewrite, RequestResolver, ResolvedAction}; use crate::response_body::ProxyResponseBody; -use crate::s3::pagination::{self, paginate}; use crate::s3::response::{ErrorResponse, ListBucketResult, ListCommonPrefix, ListContents}; use crate::types::{BucketConfig, S3Operation}; use bytes::Bytes; use http::{HeaderMap, Method}; -use object_store::ObjectStore; +use object_store::list::PaginatedListOptions; +use std::borrow::Cow; use std::time::Duration; use url::Url; use uuid::Uuid; @@ -393,14 +393,17 @@ where }) } - /// LIST via object_store + /// LIST via object_store's `PaginatedListStore`. + /// + /// Pagination is pushed to the backend — only one page of results is fetched + /// per request, avoiding loading all objects into memory. async fn handle_list( &self, config: &BucketConfig, raw_query: Option<&str>, list_rewrite: Option<&ListRewrite>, ) -> Result { - let store = self.backend.create_store(config)?; + let store = self.backend.create_paginated_store(config)?; // Parse all query parameters in a single pass let list_params = parse_list_query_params(raw_query); @@ -410,33 +413,56 @@ where // Build the full prefix including backend_prefix let full_prefix = build_list_prefix(config, client_prefix); + // Map start-after to raw key space by prepending backend_prefix + let offset = list_params + .start_after + .as_ref() + .map(|sa| build_list_prefix(config, sa)); + tracing::debug!( full_prefix = %full_prefix, delimiter = %delimiter, - "LIST via object_store" + max_keys = list_params.max_keys, + has_page_token = list_params.continuation_token.is_some(), + "LIST via PaginatedListStore" ); - let prefix_path = if full_prefix.is_empty() { + let prefix = if full_prefix.is_empty() { None } else { - Some(object_store::path::Path::from(full_prefix.as_str())) + Some(full_prefix.as_str()) + }; + + let opts = PaginatedListOptions { + offset, + delimiter: Some(Cow::Owned(delimiter.clone())), + max_keys: Some(list_params.max_keys), + page_token: list_params.continuation_token.clone(), + ..Default::default() }; - let list_result = store - .list_with_delimiter(prefix_path.as_ref()) + let paginated = store + .list_paginated(prefix, opts) .await .map_err(ProxyError::from_object_store_error)?; - // Build S3 XML response from ListResult - let bucket_name = &config.name; + // Build S3 XML response from paginated result + let key_count = paginated.result.objects.len() + paginated.result.common_prefixes.len(); let xml = build_list_xml( - bucket_name, - client_prefix, - delimiter, - &list_result, + &ListXmlParams { + bucket_name: &config.name, + client_prefix, + delimiter, + max_keys: list_params.max_keys, + is_truncated: paginated.page_token.is_some(), + key_count, + start_after: &list_params.start_after, + continuation_token: &list_params.continuation_token, + next_continuation_token: paginated.page_token, + }, + &paginated.result, config, list_rewrite, - &list_params.pagination, )?; let mut resp_headers = HeaderMap::new(); @@ -613,15 +639,28 @@ fn build_list_prefix(config: &BucketConfig, client_prefix: &str) -> String { } } +/// Parameters for building the S3 ListObjectsV2 XML response. +struct ListXmlParams<'a> { + bucket_name: &'a str, + client_prefix: &'a str, + delimiter: &'a str, + max_keys: usize, + is_truncated: bool, + key_count: usize, + start_after: &'a Option, + continuation_token: &'a Option, + next_continuation_token: Option, +} + /// Build S3 ListObjectsV2 XML from an object_store ListResult. +/// +/// Pagination is handled by the backend — `is_truncated` and +/// `next_continuation_token` are passed through from the backend's response. fn build_list_xml( - bucket_name: &str, - client_prefix: &str, - delimiter: &str, + params: &ListXmlParams<'_>, list_result: &object_store::ListResult, config: &BucketConfig, list_rewrite: Option<&ListRewrite>, - params: &pagination::PaginationParams, ) -> Result { let backend_prefix = config .backend_prefix @@ -663,21 +702,19 @@ fn build_list_xml( }) .collect(); - let page = paginate(contents, common_prefixes, params)?; - Ok(ListBucketResult { xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", - name: bucket_name.to_string(), - prefix: client_prefix.to_string(), - delimiter: delimiter.to_string(), + name: params.bucket_name.to_string(), + prefix: params.client_prefix.to_string(), + delimiter: params.delimiter.to_string(), max_keys: params.max_keys, - is_truncated: page.is_truncated, - key_count: page.contents.len() + page.common_prefixes.len(), + is_truncated: params.is_truncated, + key_count: params.key_count, start_after: params.start_after.clone(), continuation_token: params.continuation_token.clone(), - next_continuation_token: page.next_continuation_token, - contents: page.contents, - common_prefixes: page.common_prefixes, + next_continuation_token: params.next_continuation_token.clone(), + contents, + common_prefixes, } .to_xml()) } @@ -722,7 +759,9 @@ fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite> struct ListQueryParams { prefix: String, delimiter: String, - pagination: pagination::PaginationParams, + max_keys: usize, + continuation_token: Option, + start_after: Option, } /// Parse prefix, delimiter, and pagination params from a LIST query string in one pass. @@ -749,14 +788,12 @@ fn parse_list_query_params(raw_query: Option<&str>) -> ListQueryParams { ListQueryParams { prefix: prefix.unwrap_or_default(), delimiter: delimiter.unwrap_or_else(|| "/".to_string()), - pagination: pagination::PaginationParams { - max_keys: max_keys - .and_then(|v| v.parse().ok()) - .unwrap_or(1000) - .min(1000), - continuation_token, - start_after, - }, + max_keys: max_keys + .and_then(|v| v.parse().ok()) + .unwrap_or(1000) + .min(1000), + continuation_token, + start_after, } } diff --git a/crates/runtimes/cf-workers/src/client.rs b/crates/runtimes/cf-workers/src/client.rs index 383b66e..97fe026 100644 --- a/crates/runtimes/cf-workers/src/client.rs +++ b/crates/runtimes/cf-workers/src/client.rs @@ -7,12 +7,12 @@ use crate::fetch_connector::FetchConnector; use bytes::Bytes; use http::HeaderMap; +use object_store::list::PaginatedListStore; use object_store::signer::Signer; -use object_store::ObjectStore; use serde::de::DeserializeOwned; use source_coop_api::api::{CacheOptions, HttpClient}; use source_coop_core::backend::{ - build_object_store, build_signer, ProxyBackend, RawResponse, StoreBuilder, + build_paginated_list_store, build_signer, ProxyBackend, RawResponse, StoreBuilder, }; use source_coop_core::error::ProxyError; use source_coop_core::types::BucketConfig; @@ -126,8 +126,11 @@ impl HttpClient for WorkerHttpClient { pub struct WorkerBackend; impl ProxyBackend for WorkerBackend { - fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { - build_object_store(config, |b| match b { + fn create_paginated_store( + &self, + config: &BucketConfig, + ) -> Result, ProxyError> { + build_paginated_list_store(config, |b| match b { StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), }) } diff --git a/crates/runtimes/server/src/client.rs b/crates/runtimes/server/src/client.rs index 66bb32b..a650248 100644 --- a/crates/runtimes/server/src/client.rs +++ b/crates/runtimes/server/src/client.rs @@ -2,9 +2,11 @@ use bytes::Bytes; use http::HeaderMap; +use object_store::list::PaginatedListStore; use object_store::signer::Signer; -use object_store::ObjectStore; -use source_coop_core::backend::{build_object_store, build_signer, ProxyBackend, RawResponse}; +use source_coop_core::backend::{ + build_paginated_list_store, build_signer, ProxyBackend, RawResponse, +}; use source_coop_core::error::ProxyError; use source_coop_core::types::BucketConfig; use source_coop_oidc_provider::{HttpExchange, OidcProviderError}; @@ -42,8 +44,11 @@ impl Default for ServerBackend { } impl ProxyBackend for ServerBackend { - fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { - build_object_store(config, |b| b) + fn create_paginated_store( + &self, + config: &BucketConfig, + ) -> Result, ProxyError> { + build_paginated_list_store(config, |b| b) } fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { From 3cecbb8bcf755fc4dbdd80877a241668cef30048 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 3 Mar 2026 11:42:43 -0800 Subject: [PATCH 82/82] docs: object_store -> obstore --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index d98b188..49cbf6a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,7 +20,7 @@ features: - title: Unified Interface details: One stable URL per dataset, regardless of which object storage provider hosts the bytes. Backend migrations are invisible to data consumers. - title: Native S3 Compatibility - details: Works with aws-cli, boto3, DuckDB, the object_store crate, GDAL, and any S3-compatible client. No custom SDK — just set the endpoint URL. + details: Works with aws-cli, boto3, DuckDB, the obstore, GDAL, and any S3-compatible client. No custom SDK — just set the endpoint URL. - title: Metered Access details: Enforce per-identity rate limits so open data stays free for humans while protecting infrastructure from runaway machine access and egress costs. - title: Flexible Auth