From 6529a23422067d6712c98e982a70c187c1b99ac5 Mon Sep 17 00:00:00 2001 From: TheMeinerLP Date: Sun, 19 Jul 2026 12:32:45 +0200 Subject: [PATCH 1/2] fix: don't set global credential system properties for third-party S3 targets For third-party/S3-compatible endpoints (e.g. Ceph RGW), credentials already travel scoped to the filesystem instance via the s3x:// URI's userInfo. The code additionally wrote them to the process-wide aws.accessKeyId/aws.secretAccessKey system properties unconditionally, which is redundant there and unsafe: any other credential resolution in the same JVM that falls back to the default AWS credentials chain (observed: the async read-ahead path, S3ReadAheadByteChannel) would read whatever value this method last wrote, independent of which storage's request is actually in flight. Observed in production against Ceph RGW: intermittent, unpredictable 403s isolated to the async read-ahead path, with the rejected requests carrying no identifiable user at all - consistent with a credential resolution racing against an overwritten system property rather than a real permissions problem (bucket ownership and the access key itself were verified correct on the RGW side). Left the system-property assignment in place for the real-AWS-S3 branch, which has no URI-based credential mechanism and still needs it. This is a hypothesis-driven fix based on production log analysis (RGW access logs, bucket/user state, and code review) - not confirmed against a live repro, since the failure is intermittent. Worth monitoring after merge. --- .../s3/storage/S3FileSystemFactory.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java index 9a4827c..1f514f8 100644 --- a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java @@ -45,8 +45,6 @@ public static S3Fs build(S3Configuration cfg) { final URI uri; String region = resolved.region() != null && !resolved.region().isBlank() ? resolved.region() : DEFAULT_AWS_REGION; System.setProperty(AWS_REGION_KEY, region); - System.setProperty("aws.accessKeyId", cfg.getAccessKeyId()); - System.setProperty("aws.secretAccessKey", cfg.getSecretAccessKey()); // AWS SDK for Java 2.30.0+ defaults to attaching a flexible checksum (e.g. a CRC32 // trailer) to every PutObject and validating one on every GetObject. Many // third-party S3-compatible stores (Ceph RGW included) don't handle that request @@ -64,10 +62,27 @@ public static S3Fs build(S3Configuration cfg) { System.setProperty("s3.spi.force-path-style", "true"); } PROVIDER = new S3XFileSystemProvider(); + // Credentials travel in the URI's userInfo below, scoped to this one + // filesystem - deliberately NOT also written to the aws.accessKeyId/ + // aws.secretAccessKey system properties here. Those are process-wide mutable + // state; any other credential resolution in this JVM that reads the default + // AWS credentials chain (e.g. an async/background client the nio-spi-s3 + // library spins up internally, distinct from the client that honors the URI) + // would see whatever this storage's config last wrote there, independent of + // which storage's requests are actually in flight at that moment. Observed in + // production as intermittent, unpredictable 403s specifically on the async + // read-ahead path (S3ReadAheadByteChannel) against a third-party store (Ceph + // RGW), with the rejected requests showing no identifiable user at all - + // consistent with a credential resolution racing against a value this method + // (or another instance of it) had already overwritten. String userInfo = buildUserInfo(cfg); uri = new URI("s3x", userInfo, url.getHost(), url.getPort(), "/" + cfg.getBucketName(), null, null); } else { - // AWS S3 – the provider uses the default region/credentials chain + // AWS S3 – the provider uses the default region/credentials chain, which has + // no URI-embedded alternative for this code path, so these two properties are + // the only way to hand it credentials. + System.setProperty("aws.accessKeyId", cfg.getAccessKeyId()); + System.setProperty("aws.secretAccessKey", cfg.getSecretAccessKey()); PROVIDER = new S3FileSystemProvider(); uri = new URI("s3","/" + cfg.getBucketName(), null, null); From fa927c5274d0277bfc8ac2bca1d9e986b253d86f Mon Sep 17 00:00:00 2001 From: TheMeinerLP Date: Sun, 19 Jul 2026 12:32:45 +0200 Subject: [PATCH 2/2] docs: trim inline comment, keep full reasoning in the PR description The detailed why (production symptoms, ruled-out causes, the specific async-path theory) belongs in the PR, not as a wall of comments in the code itself. --- .../s3/storage/S3FileSystemFactory.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java index 1f514f8..2ce1c6d 100644 --- a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java @@ -62,25 +62,12 @@ public static S3Fs build(S3Configuration cfg) { System.setProperty("s3.spi.force-path-style", "true"); } PROVIDER = new S3XFileSystemProvider(); - // Credentials travel in the URI's userInfo below, scoped to this one - // filesystem - deliberately NOT also written to the aws.accessKeyId/ - // aws.secretAccessKey system properties here. Those are process-wide mutable - // state; any other credential resolution in this JVM that reads the default - // AWS credentials chain (e.g. an async/background client the nio-spi-s3 - // library spins up internally, distinct from the client that honors the URI) - // would see whatever this storage's config last wrote there, independent of - // which storage's requests are actually in flight at that moment. Observed in - // production as intermittent, unpredictable 403s specifically on the async - // read-ahead path (S3ReadAheadByteChannel) against a third-party store (Ceph - // RGW), with the rejected requests showing no identifiable user at all - - // consistent with a credential resolution racing against a value this method - // (or another instance of it) had already overwritten. + // Credentials go through the URI userInfo instead of the global + // aws.accessKeyId/aws.secretAccessKey properties - see PR #83. String userInfo = buildUserInfo(cfg); uri = new URI("s3x", userInfo, url.getHost(), url.getPort(), "/" + cfg.getBucketName(), null, null); } else { - // AWS S3 – the provider uses the default region/credentials chain, which has - // no URI-embedded alternative for this code path, so these two properties are - // the only way to hand it credentials. + // AWS S3: no URI-embedded credentials, so these are the only way to set them. System.setProperty("aws.accessKeyId", cfg.getAccessKeyId()); System.setProperty("aws.secretAccessKey", cfg.getSecretAccessKey()); PROVIDER = new S3FileSystemProvider();