diff --git a/Cargo.toml b/Cargo.toml index 2b5d94fb1..eac1e5872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,7 +167,7 @@ all = [ "oss", "cos", ] -azure = ["opendal/services-azblob", "reqsign", "reqwest"] +azure = ["opendal/services-azblob", "opendal/services-http", "reqsign", "reqwest"] cos = ["opendal/services-cos", "reqsign", "reqwest"] default = ["all"] gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"] diff --git a/docs/Azure.md b/docs/Azure.md index c218defd2..016e47616 100644 --- a/docs/Azure.md +++ b/docs/Azure.md @@ -6,4 +6,8 @@ the container for you - you'll need to do that yourself. You can also define a prefix that will be prepended to the keys of all cache objects created and read within the container, effectively creating a scope. To do that use the `SCCACHE_AZURE_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. +Alternatively, the `SCCACHE_AZURE_NO_CREDENTIALS` environment variable can be set to use public readonly access to the Azure Blob Storage container, without the need for credentials. Valid values for this environment variable are `true`, `1`, `false`, and `0`. + +When using anonymous access, the connection string only needs to contain the endpoint, e.g. `BlobEndpoint=https://accountname.blob.core.windows.net`. + **Important:** The environment variables are only taken into account when the server starts, i.e. only on the first run. diff --git a/docs/Configuration.md b/docs/Configuration.md index 57f3682e6..b651399c8 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -47,6 +47,8 @@ connection_string = "BlobEndpoint=https://example.blob.core.windows.net/;SharedA container = "my_container_name" # Optional string to prepend to each blob storage key key_prefix = "" +# Set to true to use anonymous access (no credentials required) +no_credentials = false [cache.disk] dir = "/tmp/.cache/sccache" @@ -236,6 +238,7 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`. * `SCCACHE_AZURE_CONNECTION_STRING` * `SCCACHE_AZURE_BLOB_CONTAINER` * `SCCACHE_AZURE_KEY_PREFIX` +* `SCCACHE_AZURE_NO_CREDENTIALS` #### gha diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 7b5a83b47..6dc5328ee 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -17,6 +17,7 @@ use opendal::Operator; use opendal::layers::{HttpClientLayer, LoggingLayer}; use opendal::services::Azblob; +use opendal::services::Http; use crate::errors::*; @@ -24,8 +25,37 @@ use super::http_client::set_user_agent; pub struct AzureBlobCache; +/// Parse the `BlobEndpoint` value from an Azure Storage connection string. +fn blob_endpoint_from_connection_string(connection_string: &str) -> Result { + for part in connection_string.split(';') { + let part = part.trim(); + if let Some(value) = part.strip_prefix("BlobEndpoint=") { + return Ok(value.to_string()); + } + } + bail!("connection string does not contain a BlobEndpoint") +} + impl AzureBlobCache { - pub fn build(connection_string: &str, container: &str, key_prefix: &str) -> Result { + pub fn build( + connection_string: &str, + container: &str, + key_prefix: &str, + no_credentials: bool, + ) -> Result { + if no_credentials { + Self::build_http_readonly(connection_string, container, key_prefix) + } else { + Self::build_azblob(connection_string, container, key_prefix) + } + } + + /// Build an operator using the OpenDAL Azblob service (authenticated). + fn build_azblob( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { let builder = Azblob::from_connection_string(connection_string)? .container(container) .root(key_prefix); @@ -36,4 +66,71 @@ impl AzureBlobCache { .finish(); Ok(op) } + + /// Build an operator using the OpenDAL HTTP service for anonymous + /// read-only access. The endpoint is constructed from the connection + /// string's `BlobEndpoint` value plus the container name, so that + /// reads go directly to + /// `https://.blob.core.windows.net//`. + fn build_http_readonly( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { + let blob_endpoint = blob_endpoint_from_connection_string(connection_string)?; + let endpoint = format!("{}/{}", blob_endpoint.trim_end_matches('/'), container); + + let builder = Http::default().endpoint(&endpoint).root(key_prefix); + + let op = Operator::new(builder)? + .layer(HttpClientLayer::new(set_user_agent())) + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_blob_endpoint() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_from_full_connection_string() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=abc123;BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_missing() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount"; + assert!(blob_endpoint_from_connection_string(cs).is_err()); + } + + #[test] + fn test_build_no_credentials() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true).unwrap(); + let info = op.info(); + assert_eq!(info.scheme(), "http"); + assert_eq!(info.root(), "/prefix/"); + } + + #[test] + fn test_build_no_credentials_missing_endpoint() { + let cs = "AccountName=myaccount"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true); + assert!(op.is_err()); + } } diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 906c69f5f..51c5fa35f 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -305,10 +305,12 @@ pub fn build_single_cache( connection_string, container, key_prefix, + no_credentials, }) => { debug!("Init azure cache with container {container}, key_prefix {key_prefix}"); - let operator = AzureBlobCache::build(connection_string, container, key_prefix) - .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; + let operator = + AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) + .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) } diff --git a/src/config.rs b/src/config.rs index 06c7dc875..7a1581c0c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -185,7 +185,10 @@ impl HTTPUrl { pub struct AzureCacheConfig { pub connection_string: String, pub container: String, + #[serde(default)] pub key_prefix: String, + #[serde(default)] + pub no_credentials: bool, } /// Configuration switches for preprocessor cache mode. @@ -910,10 +913,12 @@ fn config_from_env() -> Result { env::var("SCCACHE_AZURE_BLOB_CONTAINER"), ) { let key_prefix = key_prefix_from_env_var("SCCACHE_AZURE_KEY_PREFIX"); + let no_credentials = bool_from_env_var("SCCACHE_AZURE_NO_CREDENTIALS")?.unwrap_or(false); Some(AzureCacheConfig { connection_string, container, key_prefix, + no_credentials, }) } else { None @@ -1429,6 +1434,7 @@ fn config_overrides() { connection_string: String::new(), container: String::new(), key_prefix: String::new(), + no_credentials: false, }), disk: Some(DiskCacheConfig { dir: "/env-cache".into(), @@ -2597,3 +2603,32 @@ fn test_integration_env_variable_to_strip() { let output2 = strip_basedirs(input2, &config.basedirs); assert_eq!(&*output2, b"# 1 \"obj/file.o\""); } + +#[test] +fn test_azure_config_deserializes_without_optional_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!( + config.connection_string, + "DefaultEndpointsProtocol=https;AccountName=test" + ); + assert_eq!(config.container, "my-container"); + assert_eq!(config.key_prefix, ""); + assert!(!config.no_credentials); +} + +#[test] +fn test_azure_config_deserializes_with_all_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +key_prefix = "prefix/" +no_credentials = true +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!(config.key_prefix, "prefix/"); + assert!(config.no_credentials); +}