From eb58d245518106a2b2d56ca427d94eddc83fcd83 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 11 Feb 2026 16:12:08 -0800 Subject: [PATCH 1/3] Add support for anonymous Azure blob access The current support for Azure blob storage can't deal with the situation where the blob store needs to be accessed anonymously. This PR adds the opendal http backend and an environment variable that can be used to enable Azure anonymous access. If the environment variable is set, sccache falls back to direct http access instead of the opendal authentication flow. --- Cargo.toml | 2 +- docs/Azure.md | 4 ++ docs/Configuration.md | 3 ++ src/cache/azure.rs | 87 ++++++++++++++++++++++++++++++++++++++++++- src/cache/cache.rs | 3 +- src/config.rs | 32 ++++++++++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2b5d94fb1..eac1e5872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,7 +167,7 @@ all = [ "oss", "cos", ] -azure = ["opendal/services-azblob", "reqsign", "reqwest"] +azure = ["opendal/services-azblob", "opendal/services-http", "reqsign", "reqwest"] cos = ["opendal/services-cos", "reqsign", "reqwest"] default = ["all"] gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"] diff --git a/docs/Azure.md b/docs/Azure.md index c218defd2..016e47616 100644 --- a/docs/Azure.md +++ b/docs/Azure.md @@ -6,4 +6,8 @@ the container for you - you'll need to do that yourself. You can also define a prefix that will be prepended to the keys of all cache objects created and read within the container, effectively creating a scope. To do that use the `SCCACHE_AZURE_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. +Alternatively, the `SCCACHE_AZURE_NO_CREDENTIALS` environment variable can be set to use public readonly access to the Azure Blob Storage container, without the need for credentials. Valid values for this environment variable are `true`, `1`, `false`, and `0`. + +When using anonymous access, the connection string only needs to contain the endpoint, e.g. `BlobEndpoint=https://accountname.blob.core.windows.net`. + **Important:** The environment variables are only taken into account when the server starts, i.e. only on the first run. diff --git a/docs/Configuration.md b/docs/Configuration.md index 57f3682e6..b651399c8 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -47,6 +47,8 @@ connection_string = "BlobEndpoint=https://example.blob.core.windows.net/;SharedA container = "my_container_name" # Optional string to prepend to each blob storage key key_prefix = "" +# Set to true to use anonymous access (no credentials required) +no_credentials = false [cache.disk] dir = "/tmp/.cache/sccache" @@ -236,6 +238,7 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`. * `SCCACHE_AZURE_CONNECTION_STRING` * `SCCACHE_AZURE_BLOB_CONTAINER` * `SCCACHE_AZURE_KEY_PREFIX` +* `SCCACHE_AZURE_NO_CREDENTIALS` #### gha diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 7b5a83b47..651c41e91 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -17,6 +17,7 @@ use opendal::Operator; use opendal::layers::{HttpClientLayer, LoggingLayer}; use opendal::services::Azblob; +use opendal::services::Http; use crate::errors::*; @@ -24,8 +25,37 @@ use super::http_client::set_user_agent; pub struct AzureBlobCache; +/// Parse the `BlobEndpoint` value from an Azure Storage connection string. +fn blob_endpoint_from_connection_string(connection_string: &str) -> Result { + for part in connection_string.split(';') { + let part = part.trim(); + if let Some(value) = part.strip_prefix("BlobEndpoint=") { + return Ok(value.to_string()); + } + } + bail!("connection string does not contain a BlobEndpoint") +} + impl AzureBlobCache { - pub fn build(connection_string: &str, container: &str, key_prefix: &str) -> Result { + pub fn build( + connection_string: &str, + container: &str, + key_prefix: &str, + no_credentials: bool, + ) -> Result { + if no_credentials { + Self::build_http_readonly(connection_string, container, key_prefix) + } else { + Self::build_azblob(connection_string, container, key_prefix) + } + } + + /// Build an operator using the OpenDAL Azblob service (authenticated). + fn build_azblob( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { let builder = Azblob::from_connection_string(connection_string)? .container(container) .root(key_prefix); @@ -36,4 +66,59 @@ impl AzureBlobCache { .finish(); Ok(op) } + + /// Build an operator using the OpenDAL HTTP service for anonymous + /// read-only access. The endpoint is constructed from the connection + /// string's `BlobEndpoint` value plus the container name, so that + /// reads go directly to + /// `https://.blob.core.windows.net//`. + fn build_http_readonly( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { + let blob_endpoint = blob_endpoint_from_connection_string(connection_string)?; + let endpoint = format!( + "{}/{}", + blob_endpoint.trim_end_matches('/'), + container + ); + + let builder = Http::default().endpoint(&endpoint).root(key_prefix); + + let op = Operator::new(builder)? + .layer(HttpClientLayer::new(set_user_agent())) + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_blob_endpoint() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_from_full_connection_string() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=abc123;BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_missing() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount"; + assert!(blob_endpoint_from_connection_string(cs).is_err()); + } } diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 906c69f5f..c4e2371aa 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -305,9 +305,10 @@ pub fn build_single_cache( connection_string, container, key_prefix, + no_credentials, }) => { debug!("Init azure cache with container {container}, key_prefix {key_prefix}"); - let operator = AzureBlobCache::build(connection_string, container, key_prefix) + let operator = AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) diff --git a/src/config.rs b/src/config.rs index 06c7dc875..8bcdaf1b1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -185,7 +185,10 @@ impl HTTPUrl { pub struct AzureCacheConfig { pub connection_string: String, pub container: String, + #[serde(default)] pub key_prefix: String, + #[serde(default)] + pub no_credentials: bool, } /// Configuration switches for preprocessor cache mode. @@ -910,10 +913,12 @@ fn config_from_env() -> Result { env::var("SCCACHE_AZURE_BLOB_CONTAINER"), ) { let key_prefix = key_prefix_from_env_var("SCCACHE_AZURE_KEY_PREFIX"); + let no_credentials = bool_from_env_var("SCCACHE_AZURE_NO_CREDENTIALS")?.unwrap_or(false); Some(AzureCacheConfig { connection_string, container, key_prefix, + no_credentials, }) } else { None @@ -1429,6 +1434,7 @@ fn config_overrides() { connection_string: String::new(), container: String::new(), key_prefix: String::new(), + no_credentials: false, }), disk: Some(DiskCacheConfig { dir: "/env-cache".into(), @@ -2597,3 +2603,29 @@ fn test_integration_env_variable_to_strip() { let output2 = strip_basedirs(input2, &config.basedirs); assert_eq!(&*output2, b"# 1 \"obj/file.o\""); } + +#[test] +fn test_azure_config_deserializes_without_optional_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!(config.connection_string, "DefaultEndpointsProtocol=https;AccountName=test"); + assert_eq!(config.container, "my-container"); + assert_eq!(config.key_prefix, ""); + assert!(!config.no_credentials); +} + +#[test] +fn test_azure_config_deserializes_with_all_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +key_prefix = "prefix/" +no_credentials = true +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!(config.key_prefix, "prefix/"); + assert!(config.no_credentials); +} From 6ced09162807a51ea0761b88f7d10ef7dce3b34c Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Thu, 12 Feb 2026 10:43:27 -0800 Subject: [PATCH 2/3] Fix formatting --- src/cache/azure.rs | 6 +----- src/cache/cache.rs | 5 +++-- src/config.rs | 5 ++++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 651c41e91..1fa357eea 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -78,11 +78,7 @@ impl AzureBlobCache { key_prefix: &str, ) -> Result { let blob_endpoint = blob_endpoint_from_connection_string(connection_string)?; - let endpoint = format!( - "{}/{}", - blob_endpoint.trim_end_matches('/'), - container - ); + let endpoint = format!("{}/{}", blob_endpoint.trim_end_matches('/'), container); let builder = Http::default().endpoint(&endpoint).root(key_prefix); diff --git a/src/cache/cache.rs b/src/cache/cache.rs index c4e2371aa..51c5fa35f 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -308,8 +308,9 @@ pub fn build_single_cache( no_credentials, }) => { debug!("Init azure cache with container {container}, key_prefix {key_prefix}"); - let operator = AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) - .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; + let operator = + AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) + .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) } diff --git a/src/config.rs b/src/config.rs index 8bcdaf1b1..7a1581c0c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2611,7 +2611,10 @@ connection_string = "DefaultEndpointsProtocol=https;AccountName=test" container = "my-container" "#; let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); - assert_eq!(config.connection_string, "DefaultEndpointsProtocol=https;AccountName=test"); + assert_eq!( + config.connection_string, + "DefaultEndpointsProtocol=https;AccountName=test" + ); assert_eq!(config.container, "my-container"); assert_eq!(config.key_prefix, ""); assert!(!config.no_credentials); From 169d23df142bd7dfe184eef188ae666251075cf7 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Thu, 12 Feb 2026 10:55:45 -0800 Subject: [PATCH 3/3] Add some tests for building with anonymous access --- src/cache/azure.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 1fa357eea..6dc5328ee 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -117,4 +117,20 @@ mod tests { let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount"; assert!(blob_endpoint_from_connection_string(cs).is_err()); } + + #[test] + fn test_build_no_credentials() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true).unwrap(); + let info = op.info(); + assert_eq!(info.scheme(), "http"); + assert_eq!(info.root(), "/prefix/"); + } + + #[test] + fn test_build_no_credentials_missing_endpoint() { + let cs = "AccountName=myaccount"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true); + assert!(op.is_err()); + } }