diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index 6c46371baf..fb2c8ecb2f 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -47,8 +47,8 @@ public class EmbeddedServiceDefsUtil { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServiceDefsUtil.class); - // following servicedef list should be reviewed/updated whenever a new embedded service-def is added - public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure"; + // following servicedef list should be reviewed/updated whenever a new embedded service-def is added + public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure,s3"; private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; @@ -75,6 +75,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_OZONE_NAME = "ozone"; public static final String EMBEDDED_SERVICEDEF_KUDU_NAME = "kudu"; public static final String EMBEDDED_SERVICEDEF_NESTEDSTRUCTURE_NAME = "nestedstructure"; + public static final String EMBEDDED_SERVICEDEF_S3_NAME = "s3"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; @@ -94,6 +95,7 @@ public class EmbeddedServiceDefsUtil { public static final String TRINO_IMPL_CLASS_NAME = "org.apache.ranger.services.trino.RangerServiceTrino"; public static final String OZONE_IMPL_CLASS_NAME = "org.apache.ranger.services.ozone.RangerServiceOzone"; public static final String KUDU_IMPL_CLASS_NAME = "org.apache.ranger.services.kudu.RangerServiceKudu"; + public static final String S3_IMPL_CLASS_NAME = "org.apache.ranger.services.s3.RangerServiceS3"; private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); @@ -121,6 +123,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef ozoneServiceDef; private RangerServiceDef kuduServiceDef; private RangerServiceDef nestedStructureServiveDef; + private RangerServiceDef s3ServiceDef; private RangerServiceDef tagServiceDef; @@ -171,6 +174,7 @@ public void init(ServiceStore store) { ozoneServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_OZONE_NAME); kuduServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KUDU_NAME); nestedStructureServiveDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_NESTEDSTRUCTURE_NAME); + s3ServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_S3_NAME); // Ensure that tag service def is updated with access types of all service defs store.updateTagServiceDefForAccessTypes(); @@ -260,6 +264,8 @@ public long getElasticsearchServiceDefId() { public long getNestedStructureServiceDefId() { return getId(nestedStructureServiveDef); } + public long getS3ServiceDefId() { return getId(s3ServiceDef); } + public RangerServiceDef getEmbeddedServiceDef(String defType) throws Exception { RangerServiceDef serviceDef=null; if(StringUtils.isNotEmpty(defType)){ diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json new file mode 100644 index 0000000000..4ebd5aa6a2 --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json @@ -0,0 +1,126 @@ +{ + "id": 202, + "name": "s3", + "displayName": "s3", + "implClass": "org.apache.ranger.services.s3.RangerServiceS3", + "label": "S3", + "description": "S3 Repository", + "guid": "b8290b7f-6f69-44a9-89cc-06b6975ea676", + "resources": [ + { + "itemId": 1, + "name": "endpoint", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "S3 Endpoint", + "description": "S3 Endpoint", + "isValidLeaf": false + }, + { + "itemId": 2, + "name": "bucket", + "type": "string", + "level": 20, + "parent": "endpoint", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "S3 Bucket", + "description": "S3 Bucket", + "isValidLeaf": true + }, + { + "itemId": 3, + "name": "path", + "type": "string", + "level": 30, + "parent": "bucket", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "S3 Path", + "description": "S3 Path", + "isValidLeaf": true + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "read", + "label": "Read" + }, + { + "itemId": 2, + "name": "write", + "label": "Write" + } + ], + "configs": [ + { + "itemId": 1, + "name": "endpoint", + "type": "string", + "mandatory": true, + "label": "S3 Endpoint", + "description": "S3 Endpoint URL (e.g., s3://host:port)" + }, + { + "itemId": 2, + "name": "accesskey", + "type": "string", + "mandatory": true, + "label": "Access Key", + "description": "AWS Access Key" + }, + { + "itemId": 3, + "name": "password", + "type": "password", + "mandatory": true, + "label": "Secret Key", + "description": "AWS Secret Key" + }, + { + "itemId": 4, + "name": "region", + "type": "string", + "mandatory": false, + "defaultValue": "eu-west-1", + "label": "AWS Region", + "description": "AWS Region (default: eu-west-1)" + } + ], + "policyConditions": [ + { + "itemId": 1, + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "label": "IP Address Range", + "description": "IP Address Range" + } + ], + "dataMaskDef": {}, + "rowFilterDef": {} +} + diff --git a/dev-support/ranger-docker/scripts/create-ranger-services.py b/dev-support/ranger-docker/scripts/create-ranger-services.py index 45d6a77915..11ef0b3ddc 100644 --- a/dev-support/ranger-docker/scripts/create-ranger-services.py +++ b/dev-support/ranger-docker/scripts/create-ranger-services.py @@ -65,7 +65,15 @@ def service_not_exists(service): 'ozone.om.http-address': 'http://om:9874', 'hadoop.security.authentication': 'simple'}}) -services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone] +s3 = RangerService({'name': 'dev_s3', + 'type': 's3', + 'displayName': 'dev_s3', + 'configs': {'endpoint': 's3://localhost:9000', + 'accesskey': 'minioadmin', + 'password': 'minioadmin', + 'region': 'us-east-1'}}) + +services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone, s3] for service in services: try: if service_not_exists(service): diff --git a/distro/pom.xml b/distro/pom.xml index b7ed9b039d..be31c22e04 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -306,6 +306,12 @@ ${project.version} provided + + io.arenadata.ranger + ranger-s3-plugin + ${project.version} + provided + io.arenadata.ranger ranger-schema-registry-plugin @@ -997,6 +1003,35 @@ + + ranger-s3-plugin + + + + maven-assembly-plugin + ${assembly.plugin.version} + + ranger-${project.version} + ../target + + + + + single + + package + + false + + src/main/assembly/plugin-s3.xml + + + + + + + + ranger-schema-registry-plugin diff --git a/distro/src/main/assembly/plugin-s3.xml b/distro/src/main/assembly/plugin-s3.xml new file mode 100644 index 0000000000..193b824697 --- /dev/null +++ b/distro/src/main/assembly/plugin-s3.xml @@ -0,0 +1,138 @@ + + + + s3-plugin + + tar.gz + + ${project.parent.name}-${project.version}-s3-plugin + true + + + true + + io.arenadata.ranger:ranger-plugins-installer + io.arenadata.ranger:credentialbuilder + + + install/lib + true + false + + com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version} + com.kstruct:gethostname4j + com.sun.jersey:jersey-bundle + com.sun.jersey:jersey-core + com.sun.jersey:jersey-client + commons-cli:commons-cli + commons-collections:commons-collections + commons-configuration:commons-configuration:jar:${commons.configuration1.version} + commons-io:commons-io:jar:${commons.io.version} + commons-lang:commons-lang + commons-logging:commons-logging:jar:${commons.logging.version} + net.java.dev.jna:jna + net.java.dev.jna:jna-platform + org.apache.commons:commons-compress:jar:${commons.compress.version} + org.apache.commons:commons-configuration2:jar:${commons.configuration.version} + org.apache.commons:commons-lang3:jar:${commons.lang3.version} + org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-common:jar:${hadoop.version} + org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version} + org.slf4j:slf4j-api:jar:${slf4j-api.version} + + + + + + true + + io.arenadata.ranger:ranger-plugins-audit + io.arenadata.ranger:ranger-plugins-cred + io.arenadata.ranger:ranger-plugins-common + io.arenadata.ranger:ranger-s3-plugin + + + lib + true + false + 755 + 644 + + org.apache.commons:commons-configuration2:jar:${commons.configuration.version} + commons-configuration:commons-configuration:jar:${commons.configuration1.version} + org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version} + org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version} + org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version} + com.sun.jersey:jersey-core + com.sun.jersey:jersey-client + com.sun.jersey:jersey-bundle + com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version} + com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version} + com.amazonaws:aws-java-sdk-s3 + com.amazonaws:aws-java-sdk-core + com.amazonaws:aws-java-sdk-kms + com.amazonaws:jmespath-java + commons-lang:commons-lang + com.kstruct:gethostname4j + net.java.dev.jna:jna + net.java.dev.jna:jna-platform + + + + + + + + install/conf.templates/enable + ../plugin-s3/conf + + *.sh + + 700 + + + + ${project.build.outputDirectory} + + version + + 444 + + + + + ${project.parent.basedir}/agents-common/scripts/enable-agent.sh + + enable-s3-plugin.sh + 755 + + + ${project.parent.basedir}/agents-common/scripts/enable-agent.sh + + disable-s3-plugin.sh + 755 + + + ${project.parent.basedir}/security-admin/scripts/ranger_credential_helper.py + + 755 + + + diff --git a/plugin-s3/conf/ranger-policymgr-ssl.xml b/plugin-s3/conf/ranger-policymgr-ssl.xml new file mode 100644 index 0000000000..3a2af49059 --- /dev/null +++ b/plugin-s3/conf/ranger-policymgr-ssl.xml @@ -0,0 +1,20 @@ + + + + + diff --git a/plugin-s3/conf/ranger-s3-audit.xml b/plugin-s3/conf/ranger-s3-audit.xml new file mode 100644 index 0000000000..3a2af49059 --- /dev/null +++ b/plugin-s3/conf/ranger-s3-audit.xml @@ -0,0 +1,20 @@ + + + + + diff --git a/plugin-s3/conf/ranger-s3-security.xml b/plugin-s3/conf/ranger-s3-security.xml new file mode 100644 index 0000000000..90d9b62d33 --- /dev/null +++ b/plugin-s3/conf/ranger-s3-security.xml @@ -0,0 +1,43 @@ + + + + + + ranger.plugin.s3.policy.rest.url + + + URL to Ranger Admin + + + + + ranger.plugin.s3.service.name + + + Name of the Ranger service containing policies for this S3 instance + + + + + ranger.plugin.s3.policy.cache.dir + + + Directory where Ranger policies are cached after successful retrieval from the source + + + diff --git a/plugin-s3/pom.xml b/plugin-s3/pom.xml new file mode 100644 index 0000000000..5ee18cfdb7 --- /dev/null +++ b/plugin-s3/pom.xml @@ -0,0 +1,81 @@ + + + + 4.0.0 + + io.arenadata.ranger + ranger + 2.6.0 + .. + + ranger-s3-plugin + jar + S3 Security Plugin + S3 Security Plugin + + UTF-8 + + + + com.amazonaws + aws-java-sdk-s3 + ${aws-java-sdk.version} + + + commons-logging + commons-logging + + + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + commons-logging + commons-logging + ${commons.logging.version} + + + io.arenadata.ranger + credentialbuilder + ${project.version} + + + io.arenadata.ranger + ranger-plugins-audit + ${project.version} + + + io.arenadata.ranger + ranger-plugins-common + ${project.version} + + + diff --git a/plugin-s3/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java b/plugin-s3/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java new file mode 100644 index 0000000000..cbaf14def0 --- /dev/null +++ b/plugin-s3/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.s3; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.s3.client.S3ResourceManager; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * RangerServiceS3 provides integration between Apache Ranger and Amazon S3 compatible storage services. + * This service implementation enables policy-based access control for S3 buckets and objects. + * + *

The service supports: + *

    + *
  • Connection validation to S3 endpoints
    • + *
  • Resource lookup for buckets and paths
    • + *
  • Integration with Ranger policy engine
    • + *
+ */ +public class RangerServiceS3 extends RangerBaseService { + + private static final Log LOG = LogFactory.getLog(RangerServiceS3.class); + + /** + * Validates the configuration for connecting to an S3 service. + * + * @return Map containing validation results with status and error messages if any + * @throws Exception if validation fails due to connection or configuration errors + */ + @Override + public Map validateConfig() throws Exception { + Map ret = new HashMap(); + String serviceName = getServiceName(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerServiceS3.validateConfig(): Service: " + serviceName); + } + + if (configs != null) { + try { + ret = S3ResourceManager.validateConfig(configs); + } catch (Exception e) { + LOG.error("Error validating S3 config for service: " + serviceName, e); + throw e; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerServiceS3.validateConfig(): Response: " + ret); + } + return ret; + } + + /** + * Looks up S3 resources (buckets and paths) based on the provided context. + * This method is used by the Ranger Admin UI to provide auto-complete functionality + * when creating policies. + * + * @param context ResourceLookupContext containing user input and resource hierarchy information + * @return List of matching resource names (bucket names or paths) + * @throws Exception if resource lookup fails due to connection or permission errors + */ + @Override + public List lookupResource(ResourceLookupContext context) throws Exception { + List ret = new ArrayList(); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerServiceS3.lookupResource() Context: " + context); + } + + if (context != null) { + try { + ret = S3ResourceManager.getResources(configs, context); + } catch (Exception e) { + LOG.error("Error looking up S3 resources", e); + throw e; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerServiceS3.lookupResource() Response: " + ret); + } + return ret; + } +} diff --git a/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3Client.java b/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3Client.java new file mode 100644 index 0000000000..8d342520d9 --- /dev/null +++ b/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3Client.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.s3.client; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.ObjectListing; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.PasswordUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * S3Client provides connectivity to Amazon S3 compatible storage services + * and implements resource discovery operations for Apache Ranger integration. + * + *

This client supports: + *

    + *
  • Connection testing and validation
    • + *
  • Bucket listing and filtering
    • + *
  • Path-based resource lookup with auto-complete
    • + *
  • Custom S3-compatible endpoints (e.g., MinIO, Ceph)
    • + *
+ * + *

The client uses path-style access for compatibility with S3-compatible services. + */ +public class S3Client { + private static final Log LOG = LogFactory.getLog(S3Client.class); + private static final String DEFAULT_AWS_REGION = "us-east-1"; + private static final int MAX_RESOURCE_RESULTS = 50; + + private String endpoint; + private String accessKey; + private String secretKey; + private String awsRegion; + + /** + * Constructs an S3Client with the provided configuration. + * + * @param configs Map containing connection parameters: + * - endpoint: S3 service URL (e.g., s3://localhost:9000) + * - accesskey: AWS access key or compatible credential + * - password: Encrypted secret key (will be decrypted) + * - region: AWS region (optional, defaults to us-east-1) + * @throws Exception if required configuration is missing or invalid + */ + public S3Client(Map configs) throws Exception { + this.endpoint = configs.get("endpoint"); + this.accessKey = configs.get("accesskey"); + this.secretKey = PasswordUtils.decryptPassword(configs.get("password")); + this.awsRegion = configs.getOrDefault("region", DEFAULT_AWS_REGION); + + validateConfiguration(); + } + + /** + * Validates the S3 client configuration. + * + * @throws Exception if configuration is invalid + */ + private void validateConfiguration() throws Exception { + if (this.endpoint == null || this.endpoint.isEmpty()) { + logError("Configuration 'endpoint' is required. Please provide URL in format s3://host:port"); + } + if (this.accessKey == null || this.accessKey.isEmpty()) { + logError("Configuration 'accesskey' is required"); + } + if (this.secretKey == null || this.secretKey.isEmpty()) { + logError("Configuration 'password' (secret key) is required"); + } + } + + /** + * Logs an error message and throws an exception. + * + * @param errorMessage Error message to log and throw + * @throws Exception always throws with the provided message + */ + private static void logError(String errorMessage) throws Exception { + LOG.error(errorMessage); + throw new Exception(errorMessage); + } + + /** + * Creates and configures an Amazon S3 client instance. + * + *

The client is configured with: + *

    + *
  • Static AWS credentials
    • + *
  • S3SignerType for compatibility with older S3 implementations
    • + *
  • Path-style access for custom endpoints
    • + *
+ * + * @return Configured AmazonS3 client instance + */ + private AmazonS3 getAWSClient() { + AWSCredentials credentials = new BasicAWSCredentials(this.accessKey, this.secretKey); + + // S3SignerType is required until HTTP client libraries allow raw User-Agent headers. + // Some proxies (e.g., Airlock) modify User-Agent causing signature mismatch. + ClientConfiguration conf = new ClientConfiguration(); + conf.setSignerOverride("S3SignerType"); + + AmazonS3ClientBuilder client = AmazonS3ClientBuilder + .standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withClientConfiguration(conf) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, awsRegion)); + + client.setPathStyleAccessEnabled(true); + return client.build(); + } + + /** + * Tests connectivity to the S3 service by attempting to list buckets. + * + * @return Map containing connection test results with fields: + * - connectivityStatus: true/false + * - message: Success or error message + */ + public Map connectionTest() { + Map responseData = new HashMap(); + + try { + List buckets = getAWSClient().listBuckets(); + + if (buckets == null || buckets.isEmpty()) { + responseData.put("connectivityStatus", false); + responseData.put("message", "Connection successful but no buckets found. Verify permissions."); + LOG.warn("S3 connection test: no buckets returned"); + } else { + responseData.put("connectivityStatus", true); + responseData.put("message", "Connection test successful. Found " + buckets.size() + " bucket(s)."); + if (LOG.isDebugEnabled()) { + LOG.debug("S3 connection test successful. Buckets: " + buckets.size()); + } + } + } catch (Exception e) { + responseData.put("connectivityStatus", false); + responseData.put("message", "Connection failed: " + e.getMessage()); + LOG.error("S3 connection test failed", e); + } + return responseData; + } + + /** + * Removes leading slash from user input if present. + * + * @param userInput Input string that may start with '/' + * @return String without leading slash + */ + private String removeLeadingSlash(final String userInput) { + if (userInput != null && userInput.startsWith("/")) { + return userInput.substring(1); + } + return userInput; + } + + /** + * Retrieves S3 resource paths (buckets and pseudo-directories) matching the user input. + * + *

This method implements intelligent auto-complete functionality: + *

    + *
  • Returns matching bucket names if input is a bucket prefix
    • + *
  • Returns pseudo-directories within a bucket if input includes path
    • + *
  • Results are sorted and limited to {@link #MAX_RESOURCE_RESULTS}
    • + *
+ * + * @param userInput User-provided search string (e.g., "/my-bucket" or "/my-bucket/folder/") + * @return List of matching resource paths, each starting with '/' + */ + public List getResourcePaths(final String userInput) { + Supplier> buckets = () -> getAWSClient().listBuckets().stream(); + String[] userInputSplit = removeLeadingSlash(userInput).split("/"); + String bucketFilter = userInputSplit[0]; + String subdirFilter; + + if (userInputSplit.length >= 2) { + subdirFilter = userInput.substring(removeLeadingSlash(userInput).indexOf("/") + 2); + } else { + subdirFilter = ""; + } + + List bucketsPaths = buckets + .get() + .filter(b -> b.getName().startsWith(bucketFilter)) + .flatMap(b -> { + if (subdirFilter.length() > 0 || userInput.endsWith("/")) { + return getBucketPseudoDirectories(b.getName(), subdirFilter).stream(); + } else { + return buckets.get() + .filter(sb -> sb.getName().startsWith(bucketFilter)) + .map(sb -> String.format("/%s", sb.getName())); + } + }) + .distinct() + .sorted() + .limit(MAX_RESOURCE_RESULTS) + .collect(Collectors.toList()); + + return bucketsPaths; + } + + /** + * Retrieves pseudo-directories within an S3 bucket. + * + *

S3 does not have true directories, but this method infers directory structure + * from object keys containing '/' characters. Objects with size 0 are treated as + * directory markers. + * + * @param bucket Bucket name to search within + * @param subdirFilter Optional prefix filter for subdirectories + * @return List of pseudo-directory paths in format "/bucket/path/" + */ + public List getBucketPseudoDirectories(final String bucket, final String subdirFilter) { + ObjectListing bucketObjects = getAWSClient().listObjects(bucket); + + List pseudoDirsFiltered = bucketObjects + .getObjectSummaries() + .stream() + .filter(p -> { + if (subdirFilter != null && subdirFilter.length() > 0) { + return p.getKey().startsWith(subdirFilter); + } else { + return true; + } + }) + .map(p -> { + // Objects with size 0 are typically directory markers + if (p.getSize() == 0) { + return String.format("/%s/%s", bucket, p.getKey()); + } else { + // Extract directory path from object key + int endIndex = p.getKey().contains("/") ? p.getKey().lastIndexOf("/") : 0; + if (endIndex > 0) { + return String.format("/%s/%s/", bucket, p.getKey().substring(0, endIndex)); + } else { + return String.format("/%s/", bucket); + } + } + }) + .collect(Collectors.toList()); + + return pseudoDirsFiltered; + } +} diff --git a/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3ResourceManager.java b/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3ResourceManager.java new file mode 100644 index 0000000000..1bf7692104 --- /dev/null +++ b/plugin-s3/src/main/java/org/apache/ranger/services/s3/client/S3ResourceManager.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.s3.client; + +import org.apache.ranger.plugin.service.ResourceLookupContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3ResourceManager provides static utility methods for managing S3 client instances + * and performing resource operations such as configuration validation and resource lookup. + * + *

This class acts as a facade for S3Client operations, ensuring proper error handling + * and resource management. + */ +public class S3ResourceManager { + + /** + * Creates and returns an S3Client instance configured with the provided settings. + * + * @param configs Map containing S3 connection configuration (endpoint, accesskey, password, region) + * @return Configured S3Client instance, or null if configs is null + * @throws Exception if client creation fails due to invalid configuration + */ + public static S3Client getS3Client(Map configs) throws Exception { + if (configs != null) { + return new S3Client(configs); + } else { + return null; + } + } + + /** + * Validates the S3 service configuration by attempting to establish a connection + * and perform a basic operation (list buckets). + * + * @param configs Map containing S3 connection configuration + * @return Map containing validation results with 'connectivityStatus' and 'message' fields + * @throws Exception if validation fails due to connection or authentication errors + */ + public static Map validateConfig(Map configs) throws Exception { + Map ret = new HashMap<>(); + S3Client client = getS3Client(configs); + + if (client != null) { + ret = client.connectionTest(); + } + return ret; + } + + /** + * Retrieves a list of S3 resources (buckets and paths) matching the user input from the context. + * This method is used to provide auto-complete functionality in the Ranger Admin UI. + * + * @param configs Map containing S3 connection configuration + * @param context ResourceLookupContext containing user input and resource hierarchy + * @return List of matching resource paths, or null if no matches found + * @throws Exception if resource lookup fails due to connection or permission errors + */ + public static List getResources(Map configs, ResourceLookupContext context) throws Exception { + String userInput = context.getUserInput(); + List resources = null; + final S3Client client = getS3Client(configs); + + if (client != null) { + resources = client.getResourcePaths(userInput); + } + return resources; + } +} diff --git a/pom.xml b/pom.xml index 6de9e70514..02e228eda1 100644 --- a/pom.xml +++ b/pom.xml @@ -832,6 +832,7 @@ plugin-nifi-registry plugin-ozone plugin-presto + plugin-s3 plugin-schema-registry plugin-solr plugin-sqoop @@ -989,6 +990,7 @@ agents-installer credentialbuilder plugin-ozone + plugin-s3 ranger-hive-chained-plugin-base ranger-hive-chained-plugin-hdfs ranger-hive-chained-plugin-ozone @@ -1174,6 +1176,7 @@ plugin-nifi-registry plugin-ozone plugin-presto + plugin-s3 plugin-schema-registry plugin-solr plugin-sqoop @@ -1262,6 +1265,7 @@ plugin-nifi-registry plugin-ozone plugin-presto + plugin-s3 plugin-schema-registry plugin-solr plugin-sqoop