From c964e25fcc5125199bb8e8e9052a79113ce8ef53 Mon Sep 17 00:00:00 2001 From: Ddupg Date: Mon, 2 Mar 2026 16:10:22 +0800 Subject: [PATCH] feat(services/tos): add volcengine TOS support Change-Id: I12a490356a0996c4ce4f56614da62ca0bd8849ea --- .env.example | 5 + core/Cargo.lock | 10 ++ core/Cargo.toml | 2 + core/services/tos/Cargo.toml | 36 ++++++ core/services/tos/src/backend.rs | 199 +++++++++++++++++++++++++++++ core/services/tos/src/config.rs | 210 +++++++++++++++++++++++++++++++ core/services/tos/src/core.rs | 39 ++++++ core/services/tos/src/lib.rs | 35 ++++++ core/src/lib.rs | 3 + 9 files changed, 539 insertions(+) create mode 100644 core/services/tos/Cargo.toml create mode 100644 core/services/tos/src/backend.rs create mode 100644 core/services/tos/src/config.rs create mode 100644 core/services/tos/src/core.rs create mode 100644 core/services/tos/src/lib.rs diff --git a/.env.example b/.env.example index bd6741dc6f41..df2b81e3cc14 100644 --- a/.env.example +++ b/.env.example @@ -199,3 +199,8 @@ OPENDAL_CLOUDFLARE_KV_ROOT=/path/to/dir OPENDAL_CLOUDFLARE_KV_API_TOKEN= OPENDAL_CLOUDFLARE_KV_ACCOUNT_ID= OPENDAL_CLOUDFLARE_KV_NAMESPACE_ID= +# tos +OPENDAL_TOS_BUCKET= +OPENDAL_TOS_ENDPOINT= +OPENDAL_TOS_ACCESS_KEY_ID= +OPENDAL_TOS_ACCESS_KEY_SECRET= diff --git a/core/Cargo.lock b/core/Cargo.lock index 837d3f170d75..01cfa9ae8106 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5968,6 +5968,7 @@ dependencies = [ "opendal-service-surrealdb", "opendal-service-swift", "opendal-service-tikv", + "opendal-service-tos", "opendal-service-upyun", "opendal-service-vercel-artifacts", "opendal-service-vercel-blob", @@ -7088,6 +7089,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-tos" +version = "0.55.0" +dependencies = [ + "opendal-core", + "serde", + "serde_json", +] + [[package]] name = "opendal-service-upyun" version = "0.55.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index b64fdbe08ee3..a8f05cb0f778 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -182,6 +182,7 @@ services-sqlite = ["dep:opendal-service-sqlite"] services-surrealdb = ["dep:opendal-service-surrealdb"] services-swift = ["dep:opendal-service-swift"] services-tikv = ["dep:opendal-service-tikv"] +services-tos = ["dep:opendal-service-tos"] services-upyun = ["dep:opendal-service-upyun"] services-vercel-artifacts = ["dep:opendal-service-vercel-artifacts"] services-vercel-blob = ["dep:opendal-service-vercel-blob"] @@ -289,6 +290,7 @@ opendal-service-sqlite = { path = "services/sqlite", version = "0.55.0", optiona opendal-service-surrealdb = { path = "services/surrealdb", version = "0.55.0", optional = true, default-features = false } opendal-service-swift = { path = "services/swift", version = "0.55.0", optional = true, default-features = false } opendal-service-tikv = { path = "services/tikv", version = "0.55.0", optional = true, default-features = false } +opendal-service-tos = { path = "services/tos", version = "0.55.0", optional = true, default-features = false } opendal-service-upyun = { path = "services/upyun", version = "0.55.0", optional = true, default-features = false } opendal-service-vercel-artifacts = { path = "services/vercel-artifacts", version = "0.55.0", optional = true, default-features = false } opendal-service-vercel-blob = { path = "services/vercel-blob", version = "0.55.0", optional = true, default-features = false } diff --git a/core/services/tos/Cargo.toml b/core/services/tos/Cargo.toml new file mode 100644 index 000000000000..c11607f6575f --- /dev/null +++ b/core/services/tos/Cargo.toml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +description = "Volcengine TOS service implementation for Apache OpenDAL" +name = "opendal-service-tos" + +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } diff --git a/core/services/tos/src/backend.rs b/core/services/tos/src/backend.rs new file mode 100644 index 000000000000..3e396358408b --- /dev/null +++ b/core/services/tos/src/backend.rs @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::sync::Arc; + +use crate::TosConfig; +use crate::core::TosCore; +use opendal_core::raw::*; +use opendal_core::{Builder, Capability, Result}; + +const TOS_SCHEME: &str = "tos"; + +/// Builder for Volcengine TOS service. +#[derive(Default)] +pub struct TosBuilder { + pub(super) config: TosConfig, +} + +impl Debug for TosBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TosBuilder") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl TosBuilder { + /// Set root of this backend. + /// + /// All operations will happen under this root. + pub fn root(mut self, root: &str) -> Self { + self.config.root = if root.is_empty() { + None + } else { + Some(root.to_string()) + }; + self + } + + /// Set bucket name of this backend. + pub fn bucket(mut self, bucket: &str) -> Self { + self.config.bucket = bucket.to_string(); + self + } + + /// Set endpoint of this backend. + /// + /// Endpoint must be full uri, e.g. + /// - TOS: `https://tos-cn-beijing.volces.com` + /// - TOS with region: `https://tos-{region}.volces.com` + /// + /// If user inputs endpoint without scheme like "tos-cn-beijing.volces.com", we + /// will prepend "https://" before it. + pub fn endpoint(mut self, endpoint: &str) -> Self { + if !endpoint.is_empty() { + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + } + self + } + + /// Set region of this backend. + /// + /// Region represent the signing region of this endpoint. + /// + /// If region is not set, we will try to load it from environment. + /// If still not set, default to `cn-beijing`. + pub fn region(mut self, region: &str) -> Self { + if !region.is_empty() { + self.config.region = Some(region.to_string()); + } + self + } + + /// Set access_key_id of this backend. + /// + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub fn access_key_id(mut self, v: &str) -> Self { + self.config.access_key_id = Some(v.to_string()); + self + } + + /// Set secret_access_key of this backend. + /// + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub fn secret_access_key(mut self, v: &str) -> Self { + self.config.secret_access_key = Some(v.to_string()); + self + } + + /// Set security_token of this backend. + pub fn security_token(mut self, v: &str) -> Self { + self.config.security_token = Some(v.to_string()); + self + } + + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + pub fn allow_anonymous(mut self, allow: bool) -> Self { + self.config.allow_anonymous = allow; + self + } + + /// Set bucket versioning status for this backend. + /// + /// If set to true, OpenDAL will support versioned operations like list with + /// versions, read with version, etc. + pub fn enable_versioning(mut self, enabled: bool) -> Self { + self.config.enable_versioning = enabled; + self + } +} + +impl Builder for TosBuilder { + type Config = TosConfig; + + fn build(self) -> Result { + let mut config = self.config; + let region = config + .region + .clone() + .unwrap_or_else(|| "cn-beijing".to_string()); + + if config.endpoint.is_none() { + config.endpoint = Some(format!("https://tos-{}.volces.com", region)); + } + + let endpoint = config.endpoint.clone().unwrap(); + let bucket = config.bucket.clone(); + let root = config.root.clone().unwrap_or_else(|| "/".to_string()); + + let info = { + let am = AccessorInfo::default(); + am.set_scheme(TOS_SCHEME) + .set_root(&root) + .set_name(&bucket) + .set_native_capability(Capability { + read: false, + + write: false, + + delete: false, + + list: false, + + stat: false, + + shared: false, + + ..Default::default() + }); + + am.into() + }; + + let core = TosCore { + info, + bucket, + endpoint: endpoint.clone(), + root, + }; + + Ok(TosBackend { + core: Arc::new(core), + }) + } +} + +#[derive(Debug)] +pub struct TosBackend { + core: Arc, +} + +impl Access for TosBackend { + type Reader = (); + type Writer = (); + type Lister = (); + type Deleter = (); + + fn info(&self) -> Arc { + self.core.info.clone() + } +} diff --git a/core/services/tos/src/config.rs b/core/services/tos/src/config.rs new file mode 100644 index 000000000000..bde4dde033e9 --- /dev/null +++ b/core/services/tos/src/config.rs @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; + +use opendal_core::Configurator; +use opendal_core::OperatorUri; +use opendal_core::Result; +use serde::Deserialize; +use serde::Serialize; + +use crate::backend::TosBuilder; + +/// Config for Volcengine TOS service. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct TosConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// default to `/` if not set. + pub root: Option, + /// bucket name of this backend. + /// + /// required. + pub bucket: String, + /// endpoint of this backend. + /// + /// Endpoint must be full uri, e.g. + /// - TOS: `https://tos-cn-beijing.volces.com` + /// - TOS with region: `https://tos-{region}.volces.com` + /// + /// If user inputs endpoint without scheme like "tos-cn-beijing.volces.com", we + /// will prepend "https://" before it. + pub endpoint: Option, + /// Region represent the signing region of this endpoint. + /// + /// Required if endpoint is not provided. + /// + /// - If region is set, we will take user's input first. + /// - If not, we will try to load it from environment. + /// - If still not set, default to `cn-beijing`. + pub region: Option, + /// access_key_id of this backend. + /// + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[serde(alias = "tos_access_key_id", alias = "volcengine_access_key_id")] + pub access_key_id: Option, + /// secret_access_key of this backend. + /// + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + #[serde( + alias = "tos_secret_access_key", + alias = "volcengine_secret_access_key" + )] + pub secret_access_key: Option, + /// security_token of this backend. + /// + /// This token will expire after sometime, it's recommended to set security_token + /// by hand. + #[serde(alias = "tos_security_token", alias = "volcengine_session_token")] + pub security_token: Option, + /// Disable config load so that opendal will not load config from + /// environment. + /// + /// For examples: + /// - envs like `TOS_ACCESS_KEY_ID` + pub disable_config_load: bool, + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + pub allow_anonymous: bool, + /// Enable bucket versioning for this backend. + /// + /// If set to true, OpenDAL will support versioned operations like list with + /// versions, read with version, etc. + pub enable_versioning: bool, +} + +impl Debug for TosConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TosConfig") + .field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .finish_non_exhaustive() + } +} + +impl Configurator for TosConfig { + type Builder = TosBuilder; + + fn from_uri(uri: &OperatorUri) -> Result { + let mut map = uri.options().clone(); + + if let Some(name) = uri.name() { + map.insert("bucket".to_string(), name.to_string()); + } + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + Self::from_iter(map) + } + + fn into_builder(self) -> Self::Builder { + TosBuilder { config: self } + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use super::*; + use opendal_core::Configurator; + use opendal_core::OperatorUri; + + #[test] + fn test_tos_config_original_field_names() { + let json = r#"{ + "bucket": "test-bucket", + "access_key_id": "test-key", + "secret_access_key": "test-secret", + "region": "cn-beijing", + "endpoint": "https://tos-cn-beijing.volces.com" + }"#; + + let config: TosConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.bucket, "test-bucket"); + assert_eq!(config.access_key_id, Some("test-key".to_string())); + assert_eq!(config.secret_access_key, Some("test-secret".to_string())); + assert_eq!(config.region, Some("cn-beijing".to_string())); + assert_eq!( + config.endpoint, + Some("https://tos-cn-beijing.volces.com".to_string()) + ); + } + + #[test] + fn test_tos_config_tos_prefixed_aliases() { + let json = r#"{ + "tos_access_key_id": "test-key", + "tos_secret_access_key": "test-secret", + "tos_security_token": "test-token" + }"#; + + let config: TosConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.access_key_id, Some("test-key".to_string())); + assert_eq!(config.secret_access_key, Some("test-secret".to_string())); + assert_eq!(config.security_token, Some("test-token".to_string())); + } + + #[test] + fn test_tos_config_volcengine_prefixed_aliases() { + let json = r#"{ + "volcengine_access_key_id": "test-key", + "volcengine_secret_access_key": "test-secret", + "volcengine_session_token": "test-token" + }"#; + + let config: TosConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.access_key_id, Some("test-key".to_string())); + assert_eq!(config.secret_access_key, Some("test-secret".to_string())); + assert_eq!(config.security_token, Some("test-token".to_string())); + } + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri = OperatorUri::new("tos://example-bucket/path/to/root", iter::empty()).unwrap(); + let cfg = TosConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } + + #[test] + fn from_uri_extracts_endpoint() { + let uri = OperatorUri::new( + "tos://example-bucket/path/to/root?endpoint=https%3A%2F%2Fcustom-tos-endpoint.com", + iter::empty(), + ) + .unwrap(); + let cfg = TosConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + assert_eq!( + cfg.endpoint.as_deref(), + Some("https://custom-tos-endpoint.com") + ); + } +} diff --git a/core/services/tos/src/core.rs b/core/services/tos/src/core.rs new file mode 100644 index 000000000000..220957f920c8 --- /dev/null +++ b/core/services/tos/src/core.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::sync::Arc; + +use opendal_core::raw::*; + +pub struct TosCore { + pub info: Arc, + + pub bucket: String, + pub endpoint: String, // full endpoint with scheme, e.g. https://tos-cn-beijing.volces.com + pub root: String, +} + +impl Debug for TosCore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TosCore") + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/services/tos/src/lib.rs b/core/services/tos/src/lib.rs new file mode 100644 index 000000000000..3216bc3e67f5 --- /dev/null +++ b/core/services/tos/src/lib.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![cfg_attr(docsrs, feature(doc_cfg))] +//! Volcengine TOS service implementation for Apache OpenDAL. +#![deny(missing_docs)] + +mod backend; +mod config; +mod core; + +pub use backend::TosBuilder as Tos; +pub use config::TosConfig; + +/// Default scheme for TOS service. +pub const TOS_SCHEME: &str = "tos"; + +/// Register this service into the given registry. +pub fn register_tos_service(registry: &opendal_core::OperatorRegistry) { + registry.register::(TOS_SCHEME); +} diff --git a/core/src/lib.rs b/core/src/lib.rs index f3e7a3bc2585..302fa88dc6d5 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -205,6 +205,9 @@ fn init_default_registry_inner(registry: &OperatorRegistry) { #[cfg(feature = "services-tikv")] opendal_service_tikv::register_tikv_service(registry); + #[cfg(feature = "services-tos")] + opendal_service_tos::register_tos_service(registry); + #[cfg(feature = "services-upyun")] opendal_service_upyun::register_upyun_service(registry);