Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ force-path-style: true
| `root-path` | Optional: The root path in the S3 bucket where BlueMap data will be stored | `.` |
| `force-path-style` | Optional: Force path style access for S3 (needed for MinIO) | `false` |
| `checksum-validation` | Controls when the AWS SDK attaches/validates request and response checksums (`when_required` or `when_supported`). Since SDK 2.30 the default is `when_supported`, which many S3-compatible stores (Ceph RGW included) reject with a bare 400 Bad Request. Set to `when_supported` only if you're on real AWS S3 or a provider you've confirmed handles it. | `when_required` |
| `provider` | Optional: explicitly picks the provider profile (`r2` or `generic`) instead of auto-detecting from which fields are set. Only needed if you want to be explicit; see [docs/cloudflare-r2.md](docs/cloudflare-r2.md). | (auto-detect) |
| `account-id` | Optional: Cloudflare account ID. If set and `endpoint-url` is left empty, the endpoint is derived automatically as `https://<account-id>.r2.cloudflarestorage.com` with path-style access enabled. See [docs/cloudflare-r2.md](docs/cloudflare-r2.md). | (empty) |
| `list-cache-ttl-seconds` | Optional: How long to cache the list of available maps (`mapIds()`). Directory listings are a billed operation on some providers (e.g. R2); this list rarely changes at runtime. `0` disables caching. | `0` |

Expand Down
4 changes: 4 additions & 0 deletions docs/cloudflare-r2.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ R2-specific conveniences are built into the regular S3 config instead.
and enables path-style access automatically, since R2 only supports path-style. Leave
`endpoint-url` and `force-path-style` unset when using `account-id` this way.

This is auto-detected: setting `account-id` (with no `endpoint-url`) is enough on its own.
Add `provider: "r2"` explicitly only if you want to be unambiguous about it, e.g. in a
config generated by tooling rather than typed by hand.

If you'd rather set the endpoint yourself (e.g. for a jurisdictional/EU-restricted bucket),
skip `account-id` and use `endpoint-url`/`force-path-style` directly instead, same as any
other S3-compatible provider:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/**
* A simple storage implementation for bluemap to save data into s3 storage solution.
* Copyright (C) 2025 TheMeinerLP and contributors
* <p>
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.themeinerlp.bluemap.s3.storage;

/**
* Fallback profile: passes the configured endpoint-url/force-path-style/region through
* unchanged. Always applies, and {@link ProviderProfiles} tries it last so any provider-specific
* profile gets a chance to take over first.
*/
final class GenericS3Profile implements ProviderProfile {

static final String ID = "generic";

@Override
public String id() {
return ID;
}

@Override
public boolean appliesTo(S3Configuration cfg) {
return true;
}

@Override
public ResolvedEndpoint resolve(S3Configuration cfg) {
return new ResolvedEndpoint(cfg.getEndpointUrl(), cfg.forcePathStyle(), cfg.getRegion());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/**
* A simple storage implementation for bluemap to save data into s3 storage solution.
* Copyright (C) 2025 TheMeinerLP and contributors
* <p>
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.themeinerlp.bluemap.s3.storage;

/**
* Resolves a provider's own shorthand config (e.g. Cloudflare R2's account-id) into the plain
* connection settings {@link S3FileSystemFactory} actually needs. Keeps provider-specific
* conventions out of the generic S3 engine: add a new profile here when a provider gets its
* own convenience config, register it in {@link ProviderProfiles}, and the factory itself
* never needs to change.
*/
interface ProviderProfile {

/** Stable name users can set explicitly via the provider config option, e.g. "r2". */
String id();

/**
* Whether this profile's own shorthand config is present, so it should be picked
* automatically when the user didn't set provider explicitly. Only consulted as a
* fallback - see {@link ProviderProfiles#resolve(S3Configuration)}.
*/
boolean appliesTo(S3Configuration cfg);

/** Only called once this profile has been selected, either explicitly or via {@link #appliesTo}. */
ResolvedEndpoint resolve(S3Configuration cfg);

record ResolvedEndpoint(String endpointUrl, boolean forcePathStyle, String region) {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/**
* A simple storage implementation for bluemap to save data into s3 storage solution.
* Copyright (C) 2025 TheMeinerLP and contributors
* <p>
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.themeinerlp.bluemap.s3.storage;

import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
* Selects a {@link ProviderProfile} and resolves it against the given config.
*
* <p>If {@code provider} is set explicitly, that profile is used directly, an unmistakable,
* explicit strategy choice, and an unknown name fails fast rather than silently falling back.
* If left unset, profiles are tried in order and the first one whose {@link
* ProviderProfile#appliesTo(S3Configuration)} matches wins, so existing zero-config setups
* (e.g. just setting account-id) keep working without needing to also learn about
* {@code provider}. {@link GenericS3Profile} always applies and is tried last.
*/
final class ProviderProfiles {

private static final List<ProviderProfile> PROFILES = List.of(new R2Profile(), new GenericS3Profile());

private static final Map<String, ProviderProfile> BY_ID = new LinkedHashMap<>();

static {
for (ProviderProfile profile : PROFILES) {
BY_ID.put(profile.id(), profile);
}
}

private ProviderProfiles() {}

static ProviderProfile.ResolvedEndpoint resolve(S3Configuration cfg) {
return select(cfg).resolve(cfg);
}

private static ProviderProfile select(S3Configuration cfg) {
String explicit = cfg.getProvider();
if (explicit != null && !explicit.isBlank()) {
ProviderProfile profile = BY_ID.get(explicit.trim().toLowerCase());
if (profile == null) {
throw new IllegalArgumentException(
"Unknown provider '"
+ explicit
+ "', expected one of: "
+ String.join(", ", BY_ID.keySet()));
}
return profile;
}

return PROFILES.stream()
.filter(profile -> profile.appliesTo(cfg))
.findFirst()
.orElseThrow(() -> new IllegalStateException(
"No provider profile matched (GenericS3Profile should always match)"));
}
}
49 changes: 49 additions & 0 deletions src/main/java/dev/themeinerlp/bluemap/s3/storage/R2Profile.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/**
* A simple storage implementation for bluemap to save data into s3 storage solution.
* Copyright (C) 2025 TheMeinerLP and contributors
* <p>
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.themeinerlp.bluemap.s3.storage;

/**
* Cloudflare R2: derives the endpoint from account-id so users don't have to look up/type the
* full https://&lt;account-id&gt;.r2.cloudflarestorage.com URL themselves. Applies only when
* account-id is set and no explicit endpoint-url was given, so setting endpoint-url always
* still wins (e.g. for R2's jurisdictional/EU-restricted endpoints).
*
* <p>R2 only supports path-style access, so this always forces it on regardless of
* force-path-style.
*/
final class R2Profile implements ProviderProfile {

static final String ID = "r2";

@Override
public String id() {
return ID;
}

@Override
public boolean appliesTo(S3Configuration cfg) {
boolean noExplicitEndpoint = cfg.getEndpointUrl() == null || cfg.getEndpointUrl().isBlank();
return noExplicitEndpoint && cfg.getAccountId() != null && !cfg.getAccountId().isBlank();
}

@Override
public ResolvedEndpoint resolve(S3Configuration cfg) {
String endpoint = "https://" + cfg.getAccountId() + ".r2.cloudflarestorage.com";
return new ResolvedEndpoint(endpoint, true, cfg.getRegion());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,6 @@ public sealed interface S3Configuration permits S3StorageConfiguration {

int getListCacheTtlSeconds();

String getProvider();

}
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,12 @@ public static S3Fs build(S3Configuration cfg) {
throw new IllegalArgumentException("bucketName is required");
}

final boolean noExplicitEndpoint = cfg.getEndpointUrl() == null || cfg.getEndpointUrl().isBlank();
final boolean usingAccountId = noExplicitEndpoint && cfg.getAccountId() != null && !cfg.getAccountId().isBlank();
final String effectiveEndpoint =
usingAccountId ? "https://" + cfg.getAccountId() + ".r2.cloudflarestorage.com" : cfg.getEndpointUrl();
final boolean thirdParty = effectiveEndpoint != null && !effectiveEndpoint.isBlank();
final ProviderProfile.ResolvedEndpoint resolved = ProviderProfiles.resolve(cfg);
final boolean thirdParty = resolved.endpointUrl() != null && !resolved.endpointUrl().isBlank();
try {
final URI uri;
System.setProperty(AWS_REGION_KEY, cfg.getRegion() != null ? cfg.getRegion() : DEFAULT_AWS_REGION);
String region = resolved.region() != null && !resolved.region().isBlank() ? resolved.region() : DEFAULT_AWS_REGION;
System.setProperty(AWS_REGION_KEY, region);
System.setProperty("aws.accessKeyId", cfg.getAccessKeyId());
System.setProperty("aws.secretAccessKey", cfg.getSecretAccessKey());
// AWS SDK for Java 2.30.0+ defaults to attaching a flexible checksum (e.g. a CRC32
Expand All @@ -58,13 +56,11 @@ public static S3Fs build(S3Configuration cfg) {
System.setProperty("aws.requestChecksumCalculation", cfg.getChecksumValidation());
System.setProperty("aws.responseChecksumValidation", cfg.getChecksumValidation());
if (thirdParty) {
var url = URI.create(effectiveEndpoint);
var url = URI.create(resolved.endpointUrl());
if (!url.toString().startsWith("https")) {
System.setProperty("s3.spi.endpoint-protocol", "http");
}
// R2 only supports path-style access, so force it when we derived the endpoint
// from account-id, regardless of what force-path-style is set to.
if (cfg.forcePathStyle() || usingAccountId) {
if (resolved.forcePathStyle()) {
System.setProperty("s3.spi.force-path-style", "true");
}
PROVIDER = new S3XFileSystemProvider();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ public final class S3StorageConfiguration extends StorageConfig implements S3Con
""")
private String checksumValidation = "when_required";

@Comment("""
Optional: explicitly picks which provider profile resolves this config, one of:
"r2", "generic". Leave empty to auto-detect: setting account-id below picks "r2"
automatically, otherwise "generic" (uses endpoint-url/force-path-style as-is).
Only needed if you want to be explicit rather than rely on auto-detection.
""")
private String provider = "";

@Comment("""
Optional: Cloudflare account ID. If set and endpoint-url is left empty, the
endpoint is derived automatically as https://<account-id>.r2.cloudflarestorage.com
Expand Down Expand Up @@ -163,4 +171,9 @@ public String getAccountId() {
public int getListCacheTtlSeconds() {
return Math.max(0, listCacheTtlSeconds);
}

@Override
public String getProvider() {
return provider;
}
}
Loading