diff --git a/README.md b/README.md index c967e09..be90aeb 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ force-path-style: true | `root-path` | Optional: The root path in the S3 bucket where BlueMap data will be stored | `.` | | `force-path-style` | Optional: Force path style access for S3 (needed for MinIO) | `false` | | `checksum-validation` | Controls when the AWS SDK attaches/validates request and response checksums (`when_required` or `when_supported`). Since SDK 2.30 the default is `when_supported`, which many S3-compatible stores (Ceph RGW included) reject with a bare 400 Bad Request. Set to `when_supported` only if you're on real AWS S3 or a provider you've confirmed handles it. | `when_required` | +| `provider` | Optional: explicitly picks the provider profile (`r2` or `generic`) instead of auto-detecting from which fields are set. Only needed if you want to be explicit; see [docs/cloudflare-r2.md](docs/cloudflare-r2.md). | (auto-detect) | | `account-id` | Optional: Cloudflare account ID. If set and `endpoint-url` is left empty, the endpoint is derived automatically as `https://.r2.cloudflarestorage.com` with path-style access enabled. See [docs/cloudflare-r2.md](docs/cloudflare-r2.md). | (empty) | | `list-cache-ttl-seconds` | Optional: How long to cache the list of available maps (`mapIds()`). Directory listings are a billed operation on some providers (e.g. R2); this list rarely changes at runtime. `0` disables caching. | `0` | diff --git a/docs/cloudflare-r2.md b/docs/cloudflare-r2.md index a18eada..9f724f7 100644 --- a/docs/cloudflare-r2.md +++ b/docs/cloudflare-r2.md @@ -31,6 +31,10 @@ R2-specific conveniences are built into the regular S3 config instead. and enables path-style access automatically, since R2 only supports path-style. Leave `endpoint-url` and `force-path-style` unset when using `account-id` this way. + This is auto-detected: setting `account-id` (with no `endpoint-url`) is enough on its own. + Add `provider: "r2"` explicitly only if you want to be unambiguous about it, e.g. in a + config generated by tooling rather than typed by hand. + If you'd rather set the endpoint yourself (e.g. for a jurisdictional/EU-restricted bucket), skip `account-id` and use `endpoint-url`/`force-path-style` directly instead, same as any other S3-compatible provider: diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/GenericS3Profile.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/GenericS3Profile.java new file mode 100644 index 0000000..ee690a2 --- /dev/null +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/GenericS3Profile.java @@ -0,0 +1,43 @@ +/** + * A simple storage implementation for bluemap to save data into s3 storage solution. + * Copyright (C) 2025 TheMeinerLP and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package dev.themeinerlp.bluemap.s3.storage; + +/** + * Fallback profile: passes the configured endpoint-url/force-path-style/region through + * unchanged. Always applies, and {@link ProviderProfiles} tries it last so any provider-specific + * profile gets a chance to take over first. + */ +final class GenericS3Profile implements ProviderProfile { + + static final String ID = "generic"; + + @Override + public String id() { + return ID; + } + + @Override + public boolean appliesTo(S3Configuration cfg) { + return true; + } + + @Override + public ResolvedEndpoint resolve(S3Configuration cfg) { + return new ResolvedEndpoint(cfg.getEndpointUrl(), cfg.forcePathStyle(), cfg.getRegion()); + } +} diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfile.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfile.java new file mode 100644 index 0000000..8ef5ae9 --- /dev/null +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfile.java @@ -0,0 +1,43 @@ +/** + * A simple storage implementation for bluemap to save data into s3 storage solution. + * Copyright (C) 2025 TheMeinerLP and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package dev.themeinerlp.bluemap.s3.storage; + +/** + * Resolves a provider's own shorthand config (e.g. Cloudflare R2's account-id) into the plain + * connection settings {@link S3FileSystemFactory} actually needs. Keeps provider-specific + * conventions out of the generic S3 engine: add a new profile here when a provider gets its + * own convenience config, register it in {@link ProviderProfiles}, and the factory itself + * never needs to change. + */ +interface ProviderProfile { + + /** Stable name users can set explicitly via the provider config option, e.g. "r2". */ + String id(); + + /** + * Whether this profile's own shorthand config is present, so it should be picked + * automatically when the user didn't set provider explicitly. Only consulted as a + * fallback - see {@link ProviderProfiles#resolve(S3Configuration)}. + */ + boolean appliesTo(S3Configuration cfg); + + /** Only called once this profile has been selected, either explicitly or via {@link #appliesTo}. */ + ResolvedEndpoint resolve(S3Configuration cfg); + + record ResolvedEndpoint(String endpointUrl, boolean forcePathStyle, String region) {} +} diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfiles.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfiles.java new file mode 100644 index 0000000..e9ccae4 --- /dev/null +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/ProviderProfiles.java @@ -0,0 +1,72 @@ +/** + * A simple storage implementation for bluemap to save data into s3 storage solution. + * Copyright (C) 2025 TheMeinerLP and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package dev.themeinerlp.bluemap.s3.storage; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Selects a {@link ProviderProfile} and resolves it against the given config. + * + *

If {@code provider} is set explicitly, that profile is used directly, an unmistakable, + * explicit strategy choice, and an unknown name fails fast rather than silently falling back. + * If left unset, profiles are tried in order and the first one whose {@link + * ProviderProfile#appliesTo(S3Configuration)} matches wins, so existing zero-config setups + * (e.g. just setting account-id) keep working without needing to also learn about + * {@code provider}. {@link GenericS3Profile} always applies and is tried last. + */ +final class ProviderProfiles { + + private static final List PROFILES = List.of(new R2Profile(), new GenericS3Profile()); + + private static final Map BY_ID = new LinkedHashMap<>(); + + static { + for (ProviderProfile profile : PROFILES) { + BY_ID.put(profile.id(), profile); + } + } + + private ProviderProfiles() {} + + static ProviderProfile.ResolvedEndpoint resolve(S3Configuration cfg) { + return select(cfg).resolve(cfg); + } + + private static ProviderProfile select(S3Configuration cfg) { + String explicit = cfg.getProvider(); + if (explicit != null && !explicit.isBlank()) { + ProviderProfile profile = BY_ID.get(explicit.trim().toLowerCase()); + if (profile == null) { + throw new IllegalArgumentException( + "Unknown provider '" + + explicit + + "', expected one of: " + + String.join(", ", BY_ID.keySet())); + } + return profile; + } + + return PROFILES.stream() + .filter(profile -> profile.appliesTo(cfg)) + .findFirst() + .orElseThrow(() -> new IllegalStateException( + "No provider profile matched (GenericS3Profile should always match)")); + } +} diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/R2Profile.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/R2Profile.java new file mode 100644 index 0000000..962a44e --- /dev/null +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/R2Profile.java @@ -0,0 +1,49 @@ +/** + * A simple storage implementation for bluemap to save data into s3 storage solution. + * Copyright (C) 2025 TheMeinerLP and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package dev.themeinerlp.bluemap.s3.storage; + +/** + * Cloudflare R2: derives the endpoint from account-id so users don't have to look up/type the + * full https://<account-id>.r2.cloudflarestorage.com URL themselves. Applies only when + * account-id is set and no explicit endpoint-url was given, so setting endpoint-url always + * still wins (e.g. for R2's jurisdictional/EU-restricted endpoints). + * + *

R2 only supports path-style access, so this always forces it on regardless of + * force-path-style. + */ +final class R2Profile implements ProviderProfile { + + static final String ID = "r2"; + + @Override + public String id() { + return ID; + } + + @Override + public boolean appliesTo(S3Configuration cfg) { + boolean noExplicitEndpoint = cfg.getEndpointUrl() == null || cfg.getEndpointUrl().isBlank(); + return noExplicitEndpoint && cfg.getAccountId() != null && !cfg.getAccountId().isBlank(); + } + + @Override + public ResolvedEndpoint resolve(S3Configuration cfg) { + String endpoint = "https://" + cfg.getAccountId() + ".r2.cloudflarestorage.com"; + return new ResolvedEndpoint(endpoint, true, cfg.getRegion()); + } +} diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3Configuration.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3Configuration.java index efb7afe..1803c57 100644 --- a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3Configuration.java +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3Configuration.java @@ -39,4 +39,6 @@ public sealed interface S3Configuration permits S3StorageConfiguration { int getListCacheTtlSeconds(); + String getProvider(); + } diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java index d940a48..9a4827c 100644 --- a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3FileSystemFactory.java @@ -39,14 +39,12 @@ public static S3Fs build(S3Configuration cfg) { throw new IllegalArgumentException("bucketName is required"); } - final boolean noExplicitEndpoint = cfg.getEndpointUrl() == null || cfg.getEndpointUrl().isBlank(); - final boolean usingAccountId = noExplicitEndpoint && cfg.getAccountId() != null && !cfg.getAccountId().isBlank(); - final String effectiveEndpoint = - usingAccountId ? "https://" + cfg.getAccountId() + ".r2.cloudflarestorage.com" : cfg.getEndpointUrl(); - final boolean thirdParty = effectiveEndpoint != null && !effectiveEndpoint.isBlank(); + final ProviderProfile.ResolvedEndpoint resolved = ProviderProfiles.resolve(cfg); + final boolean thirdParty = resolved.endpointUrl() != null && !resolved.endpointUrl().isBlank(); try { final URI uri; - System.setProperty(AWS_REGION_KEY, cfg.getRegion() != null ? cfg.getRegion() : DEFAULT_AWS_REGION); + String region = resolved.region() != null && !resolved.region().isBlank() ? resolved.region() : DEFAULT_AWS_REGION; + System.setProperty(AWS_REGION_KEY, region); System.setProperty("aws.accessKeyId", cfg.getAccessKeyId()); System.setProperty("aws.secretAccessKey", cfg.getSecretAccessKey()); // AWS SDK for Java 2.30.0+ defaults to attaching a flexible checksum (e.g. a CRC32 @@ -58,13 +56,11 @@ public static S3Fs build(S3Configuration cfg) { System.setProperty("aws.requestChecksumCalculation", cfg.getChecksumValidation()); System.setProperty("aws.responseChecksumValidation", cfg.getChecksumValidation()); if (thirdParty) { - var url = URI.create(effectiveEndpoint); + var url = URI.create(resolved.endpointUrl()); if (!url.toString().startsWith("https")) { System.setProperty("s3.spi.endpoint-protocol", "http"); } - // R2 only supports path-style access, so force it when we derived the endpoint - // from account-id, regardless of what force-path-style is set to. - if (cfg.forcePathStyle() || usingAccountId) { + if (resolved.forcePathStyle()) { System.setProperty("s3.spi.force-path-style", "true"); } PROVIDER = new S3XFileSystemProvider(); diff --git a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3StorageConfiguration.java b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3StorageConfiguration.java index e9a5924..5f7b9b5 100644 --- a/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3StorageConfiguration.java +++ b/src/main/java/dev/themeinerlp/bluemap/s3/storage/S3StorageConfiguration.java @@ -66,6 +66,14 @@ public final class S3StorageConfiguration extends StorageConfig implements S3Con """) private String checksumValidation = "when_required"; + @Comment(""" + Optional: explicitly picks which provider profile resolves this config, one of: + "r2", "generic". Leave empty to auto-detect: setting account-id below picks "r2" + automatically, otherwise "generic" (uses endpoint-url/force-path-style as-is). + Only needed if you want to be explicit rather than rely on auto-detection. + """) + private String provider = ""; + @Comment(""" Optional: Cloudflare account ID. If set and endpoint-url is left empty, the endpoint is derived automatically as https://.r2.cloudflarestorage.com @@ -163,4 +171,9 @@ public String getAccountId() { public int getListCacheTtlSeconds() { return Math.max(0, listCacheTtlSeconds); } + + @Override + public String getProvider() { + return provider; + } }