diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileCacheProviderSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileCacheProviderSpool.java index 810dac288cc..47ab049fb86 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileCacheProviderSpool.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileCacheProviderSpool.java @@ -149,6 +149,13 @@ public boolean init(Properties props, String basePropertyName) { "755"); filePermissions = AuditFileUtil.parsePermissions(spoolFilePerms); Set dirPermissions = AuditFileUtil.parsePermissions(spoolDirPerms); + AuditFileUtil.ResolvedDirectory resolvedLogDirectory = AuditFileUtil.resolveDirectory(logFolderProp, + MiscUtil.getStringProperty(props, propPrefix + ".subdir.mode"), + dirPermissions, + filePermissions); + logFolderProp = resolvedLogDirectory.getPath(); + dirPermissions = resolvedLogDirectory.getDirPermissions(); + filePermissions = resolvedLogDirectory.getFilePermissions(); logFileNameFormat = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); String archiveFolderProp = MiscUtil.getStringProperty(props, @@ -180,12 +187,11 @@ public boolean init(Properties props, String basePropertyName) { return false; } logFolder = new File(logFolderProp); - if (!logFolder.isDirectory()) { - AuditFileUtil.createDirectoryWithPermissions(logFolder, dirPermissions); - if (!logFolder.isDirectory()) { - logger.error("File Spool folder not found and can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), FILE_CACHE_PROVIDER_NAME); - return false; - } + try { + resolvedLogDirectory.ensureDirectory(); + } catch (Exception excp) { + logger.error("File Spool folder not found, unsafe, or can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), FILE_CACHE_PROVIDER_NAME, excp); + return false; } logger.info("logFolder=" + logFolder + ", queueName=" + FILE_CACHE_PROVIDER_NAME); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileQueueSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileQueueSpool.java index 17d2f56a3a6..3f51c3fbb99 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileQueueSpool.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileQueueSpool.java @@ -146,6 +146,13 @@ public boolean init(Properties props, String basePropertyName) { "755"); filePermissions = AuditFileUtil.parsePermissions(spoolFilePerms); Set dirPermissions = AuditFileUtil.parsePermissions(spoolDirPerms); + AuditFileUtil.ResolvedDirectory resolvedLogDirectory = AuditFileUtil.resolveDirectory(logFolderProp, + MiscUtil.getStringProperty(props, propPrefix + ".subdir.mode"), + dirPermissions, + filePermissions); + logFolderProp = resolvedLogDirectory.getPath(); + dirPermissions = resolvedLogDirectory.getDirPermissions(); + filePermissions = resolvedLogDirectory.getFilePermissions(); logFileNameFormat = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); String archiveFolderProp = MiscUtil.getStringProperty(props, @@ -176,12 +183,11 @@ public boolean init(Properties props, String basePropertyName) { return false; } logFolder = new File(logFolderProp); - if (!logFolder.isDirectory()) { - AuditFileUtil.createDirectoryWithPermissions(logFolder, dirPermissions); - if (!logFolder.isDirectory()) { - logger.error("File Spool folder not found and can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), FILE_QUEUE_PROVIDER_NAME); - return false; - } + try { + resolvedLogDirectory.ensureDirectory(); + } catch (Exception excp) { + logger.error("File Spool folder not found, unsafe, or can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), FILE_QUEUE_PROVIDER_NAME, excp); + return false; } logger.info("logFolder=" + logFolder + ", queueName=" + FILE_QUEUE_PROVIDER_NAME); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java index 3295cb93128..a386824c2e7 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java @@ -137,6 +137,13 @@ public boolean init(Properties props, String basePropertyName) { "755"); filePermissions = AuditFileUtil.parsePermissions(spoolFilePerms); Set dirPermissions = AuditFileUtil.parsePermissions(spoolDirPerms); + AuditFileUtil.ResolvedDirectory resolvedLogDirectory = AuditFileUtil.resolveDirectory(logFolderProp, + MiscUtil.getStringProperty(props, propPrefix + ".subdir.mode"), + dirPermissions, + filePermissions); + logFolderProp = resolvedLogDirectory.getPath(); + dirPermissions = resolvedLogDirectory.getDirPermissions(); + filePermissions = resolvedLogDirectory.getFilePermissions(); logFileNameFormat = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_FILE_SPOOL_LOCAL_FILE_NAME); String archiveFolderProp = MiscUtil.getStringProperty(props, @@ -161,12 +168,11 @@ public boolean init(Properties props, String basePropertyName) { return false; } logFolder = new File(logFolderProp); - if (!logFolder.isDirectory()) { - AuditFileUtil.createDirectoryWithPermissions(logFolder, dirPermissions); - if (!logFolder.isDirectory()) { - logger.error("File Spool folder not found and can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), queueProvider.getName()); - return false; - } + try { + resolvedLogDirectory.ensureDirectory(); + } catch (Exception excp) { + logger.error("File Spool folder not found, unsafe, or can't be created. folder={}, queueName={}", logFolder.getAbsolutePath(), queueProvider.getName(), excp); + return false; } logger.info("logFolder={}, queueName={}", logFolder, queueProvider.getName()); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AuditFileUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AuditFileUtil.java index cc16654821c..6da58da1b88 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AuditFileUtil.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AuditFileUtil.java @@ -28,6 +28,9 @@ import java.util.Set; public class AuditFileUtil { + public static final String SUBDIR_MODE_DISABLED = "disabled"; + public static final String SUBDIR_MODE_PERUSER = "peruser"; + public static final String SUBDIR_MODE_PERGROUP = "pergroup"; public static void setPermissions(File file, Set perms) throws IOException { Path path = file.toPath(); @@ -56,4 +59,32 @@ public static void createDirectoryWithPermissions(File dir, Set defaultDirPerms, Set defaultFilePerms) { + return new ResolvedDirectory(LocalDirectoryResolver.resolveAuditSpool(baseDir, subdirMode, defaultDirPerms, defaultFilePerms)); + } + + public static final class ResolvedDirectory { + private final LocalDirectoryResolver.ResolvedDirectory delegate; + + private ResolvedDirectory(LocalDirectoryResolver.ResolvedDirectory delegate) { + this.delegate = delegate; + } + + public String getPath() { + return delegate.getPath(); + } + + public Set getDirPermissions() { + return delegate.getDirPermissions(); + } + + public Set getFilePermissions() { + return delegate.getFilePermissions(); + } + + public void ensureDirectory() throws IOException { + delegate.ensureDirectory(); + } + } + } diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/LocalDirectoryResolver.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/LocalDirectoryResolver.java new file mode 100644 index 00000000000..d07d6570400 --- /dev/null +++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/LocalDirectoryResolver.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.audit.utils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +public final class LocalDirectoryResolver { + public static final String SUBDIR_MODE_DISABLED = "disabled"; + public static final String SUBDIR_MODE_PERUSER = "peruser"; + public static final String SUBDIR_MODE_PERGROUP = "pergroup"; + + private static final Set PERUSER_DIR_PERMS = PosixFilePermissions.fromString("rwx------"); + private static final Set PERUSER_FILE_PERMS = PosixFilePermissions.fromString("rw-------"); + private static final Set PERGROUP_DIR_PERMS = PosixFilePermissions.fromString("rwxrwx---"); + private static final Set PERGROUP_FILE_PERMS = PosixFilePermissions.fromString("rw-rw----"); + private static final int DIRECTORY_VALIDATION_RETRY_COUNT = 3; + private static final long DIRECTORY_VALIDATION_RETRY_INTERVAL_MS = 50L; + + private LocalDirectoryResolver() { + } + + public static ResolvedDirectory resolveAuditSpool(String baseDir, String subdirMode, Set defaultDirPerms, Set defaultFilePerms) { + return resolve(baseDir, subdirMode, defaultDirPerms, defaultFilePerms, "audit spool", "Audit spool path", "Base audit spool directory", "audit spool directory"); + } + + public static ResolvedDirectory resolveLocalDirectory(String baseDir, String subdirMode, Set defaultDirPerms, Set defaultFilePerms) { + return resolve(baseDir, subdirMode, defaultDirPerms, defaultFilePerms, "local directory", "Local directory", "Base local directory", "local directory"); + } + + private static ResolvedDirectory resolve(String baseDir, String subdirMode, Set defaultDirPerms, Set defaultFilePerms, String subdirDescription, String directoryLabel, String baseDirectoryLabel, String ownershipLabel) { + String mode = StringUtils.defaultIfBlank(subdirMode, SUBDIR_MODE_DISABLED).trim().toLowerCase(); + + if (SUBDIR_MODE_DISABLED.equals(mode)) { + return new ResolvedDirectory(null, baseDir, null, defaultDirPerms, defaultFilePerms, null, null, directoryLabel, baseDirectoryLabel, ownershipLabel); + } else if (SUBDIR_MODE_PERUSER.equals(mode)) { + String user = sanitizePathElement(getCurrentUser(), "user", subdirDescription); + + return new ResolvedDirectory(baseDir, appendPath(baseDir, user), defaultDirPerms, PERUSER_DIR_PERMS, PERUSER_FILE_PERMS, user, null, directoryLabel, baseDirectoryLabel, ownershipLabel); + } else if (SUBDIR_MODE_PERGROUP.equals(mode)) { + String group = sanitizePathElement(getCurrentGroup(subdirDescription), "group", subdirDescription); + + return new ResolvedDirectory(baseDir, appendPath(baseDir, group), defaultDirPerms, PERGROUP_DIR_PERMS, PERGROUP_FILE_PERMS, null, group, directoryLabel, baseDirectoryLabel, ownershipLabel); + } + + throw new IllegalArgumentException("Unsupported " + subdirDescription + " subdir mode: " + subdirMode); + } + + private static String appendPath(String baseDir, String child) { + if (StringUtils.isBlank(baseDir)) { + return baseDir; + } + + return new File(baseDir, child).getPath(); + } + + private static String sanitizePathElement(String value, String label, String subdirDescription) { + String ret = StringUtils.trim(value); + + if (StringUtils.isBlank(ret) || ".".equals(ret) || "..".equals(ret) || ret.contains("..") || + ret.indexOf('/') >= 0 || ret.indexOf('\\') >= 0 || ret.indexOf(File.separatorChar) >= 0 || + ret.indexOf(File.pathSeparatorChar) >= 0) { + throw new IllegalArgumentException("Invalid " + label + " value for " + subdirDescription + " subdir: " + value); + } + + return ret; + } + + private static String getCurrentUser() { + try { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + if (ugi != null && StringUtils.isNotBlank(ugi.getShortUserName())) { + return ugi.getShortUserName(); + } + } catch (IOException ignored) { + } + + return System.getProperty("user.name"); + } + + private static String getCurrentGroup(String subdirDescription) { + try { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + if (ugi != null && StringUtils.isNotBlank(ugi.getPrimaryGroupName())) { + return ugi.getPrimaryGroupName(); + } + } catch (IOException ignored) { + } + + throw new IllegalStateException("Unable to determine current user's primary group for " + subdirDescription + " subdir"); + } + + public static final class ResolvedDirectory { + private final String basePath; + private final String path; + private final Set baseDirPermissions; + private final Set dirPermissions; + private final Set filePermissions; + private final String expectedOwner; + private final String expectedGroup; + private final String directoryLabel; + private final String baseDirectoryLabel; + private final String ownershipLabel; + + private ResolvedDirectory(String basePath, String path, Set baseDirPermissions, Set dirPermissions, Set filePermissions, String expectedOwner, String expectedGroup, String directoryLabel, String baseDirectoryLabel, String ownershipLabel) { + this.basePath = basePath; + this.path = path; + this.baseDirPermissions = baseDirPermissions; + this.dirPermissions = dirPermissions; + this.filePermissions = filePermissions; + this.expectedOwner = expectedOwner; + this.expectedGroup = expectedGroup; + this.directoryLabel = directoryLabel; + this.baseDirectoryLabel = baseDirectoryLabel; + this.ownershipLabel = ownershipLabel; + } + + public String getPath() { + return path; + } + + public Set getDirPermissions() { + return dirPermissions; + } + + public Set getFilePermissions() { + return filePermissions; + } + + public void ensureDirectory() throws IOException { + if (StringUtils.isBlank(path)) { + return; + } + + Path directory = Paths.get(path); + + if (StringUtils.isNotBlank(basePath)) { + ensureBaseDirectory(); + } + + boolean createdByAnotherProcess = false; + + if (Files.exists(directory, LinkOption.NOFOLLOW_LINKS)) { + validateDirectory(directory, dirPermissions, expectedOwner, expectedGroup, directoryLabel, ownershipLabel); + return; + } else if (StringUtils.isNotBlank(basePath)) { + try { + Files.createDirectory(directory, PosixFilePermissions.asFileAttribute(dirPermissions)); + Files.setPosixFilePermissions(directory, dirPermissions); + } catch (FileAlreadyExistsException ignored) { + // Another local process can create the same per-user/per-group subdirectory concurrently. + createdByAnotherProcess = true; + } + } else if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS)) { + Files.createDirectories(directory, PosixFilePermissions.asFileAttribute(dirPermissions)); + Files.setPosixFilePermissions(directory, dirPermissions); + } + + if (createdByAnotherProcess) { + validateDirectoryAfterConcurrentCreate(directory, dirPermissions, expectedOwner, expectedGroup, directoryLabel, ownershipLabel); + } else { + validateDirectory(directory, dirPermissions, expectedOwner, expectedGroup, directoryLabel, ownershipLabel); + } + } + + private void ensureBaseDirectory() throws IOException { + Path baseDirectory = Paths.get(basePath); + + if (!Files.exists(baseDirectory, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException(baseDirectoryLabel + " does not exist: " + baseDirectory); + } + + validateDirectory(baseDirectory, baseDirPermissions, null, null, baseDirectoryLabel, ownershipLabel); + } + + private void validateDirectory(Path directory, Set expectedPermissions, String expectedOwner, String expectedGroup, String label, String ownershipLabel) throws IOException { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException(label + " is not a regular directory: " + directory); + } + + PosixFileAttributeView view = Files.getFileAttributeView(directory, PosixFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + + if (view != null) { + PosixFileAttributes attrs = view.readAttributes(); + + if (expectedPermissions != null && !attrs.permissions().equals(expectedPermissions)) { + throw new IOException("Unsafe permissions on " + StringUtils.lowerCase(label) + " " + directory + ": expected " + expectedPermissions + ", actual " + attrs.permissions()); + } + + if (expectedOwner != null && !StringUtils.equals(attrs.owner().getName(), expectedOwner)) { + throw new IOException("Unexpected owner on " + ownershipLabel + " " + directory + ": expected " + expectedOwner + ", actual " + attrs.owner().getName()); + } + + if (expectedGroup != null && !StringUtils.equals(attrs.group().getName(), expectedGroup)) { + throw new IOException("Unexpected group on " + ownershipLabel + " " + directory + ": expected " + expectedGroup + ", actual " + attrs.group().getName()); + } + } + } + + private void validateDirectoryAfterConcurrentCreate(Path directory, Set expectedPermissions, String expectedOwner, String expectedGroup, String label, String ownershipLabel) throws IOException { + IOException lastException = null; + + for (int attempt = 0; attempt < DIRECTORY_VALIDATION_RETRY_COUNT; attempt++) { + try { + validateDirectory(directory, expectedPermissions, expectedOwner, expectedGroup, label, ownershipLabel); + return; + } catch (IOException exception) { + lastException = exception; + + if (attempt + 1 < DIRECTORY_VALIDATION_RETRY_COUNT) { + try { + Thread.sleep(DIRECTORY_VALIDATION_RETRY_INTERVAL_MS); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + exception.addSuppressed(interruptedException); + throw exception; + } + } + } + } + + throw lastException; + } + } +} diff --git a/agents-audit/src/test/java/org/apache/ranger/audit/utils/AuditFileUtilTest.java b/agents-audit/src/test/java/org/apache/ranger/audit/utils/AuditFileUtilTest.java index 1b5b2956104..95463d25db7 100644 --- a/agents-audit/src/test/java/org/apache/ranger/audit/utils/AuditFileUtilTest.java +++ b/agents-audit/src/test/java/org/apache/ranger/audit/utils/AuditFileUtilTest.java @@ -24,11 +24,20 @@ import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.Set; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class AuditFileUtilTest { @@ -150,4 +159,178 @@ public void testCreateDirectoryWithPermissions777() throws Exception { } } + @Test + public void testResolvePerUserDirectoryUsesPrivatePermissions() throws Exception { + Path tempBaseDir = Files.createTempDirectory("auditSpoolDir"); + try { + Files.setPosixFilePermissions(tempBaseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(tempBaseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERUSER, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + + directory.ensureDirectory(); + + Path resolvedPath = new File(directory.getPath()).toPath(); + assertTrue("Directory should exist", Files.isDirectory(resolvedPath)); + assertEquals("Base directory permissions should not be changed", PosixFilePermissions.fromString("rwxr-xr-x"), Files.getPosixFilePermissions(tempBaseDir)); + assertEquals("Directory permissions should be private", PosixFilePermissions.fromString("rwx------"), Files.getPosixFilePermissions(resolvedPath)); + assertEquals("File permissions should be private", PosixFilePermissions.fromString("rw-------"), directory.getFilePermissions()); + } finally { + deleteRecursively(tempBaseDir); + } + } + + @Test + public void testResolvePerGroupDirectoryUsesGroupPermissions() throws Exception { + Path tempBaseDir = Files.createTempDirectory("auditSpoolDir"); + try { + Files.setPosixFilePermissions(tempBaseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(tempBaseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERGROUP, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + + directory.ensureDirectory(); + + Path resolvedPath = new File(directory.getPath()).toPath(); + assertTrue("Directory should exist", Files.isDirectory(resolvedPath)); + assertEquals("Base directory permissions should not be changed", PosixFilePermissions.fromString("rwxr-xr-x"), Files.getPosixFilePermissions(tempBaseDir)); + assertEquals("Directory permissions should be group-scoped", PosixFilePermissions.fromString("rwxrwx---"), Files.getPosixFilePermissions(resolvedPath)); + assertEquals("File permissions should be group-scoped", PosixFilePermissions.fromString("rw-rw----"), directory.getFilePermissions()); + } finally { + deleteRecursively(tempBaseDir); + } + } + + @Test + public void testResolvePerGroupDirectoryHandlesConcurrentCreation() throws Exception { + Path tempBaseDir = Files.createTempDirectory("auditSpoolDir"); + + try { + Files.setPosixFilePermissions(tempBaseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + final AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(tempBaseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERGROUP, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + int threadCount = 8; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + + try { + final CyclicBarrier startBarrier = new CyclicBarrier(threadCount); + List> futures = new ArrayList<>(); + + for (int i = 0; i < threadCount; i++) { + futures.add(executorService.submit(() -> { + startBarrier.await(10, TimeUnit.SECONDS); + directory.ensureDirectory(); + return null; + })); + } + + for (Future future : futures) { + future.get(10, TimeUnit.SECONDS); + } + } finally { + executorService.shutdownNow(); + assertTrue("Executor should terminate", executorService.awaitTermination(10, TimeUnit.SECONDS)); + } + + Path resolvedPath = new File(directory.getPath()).toPath(); + assertTrue("Directory should exist", Files.isDirectory(resolvedPath)); + assertEquals("Directory permissions should be group-scoped", PosixFilePermissions.fromString("rwxrwx---"), Files.getPosixFilePermissions(resolvedPath)); + } finally { + deleteRecursively(tempBaseDir); + } + } + + @Test(expected = IOException.class) + public void testResolveDirectoryRejectsUnsafeExistingPermissions() throws Exception { + Path tempBaseDir = Files.createTempDirectory("auditSpoolDir"); + try { + Files.setPosixFilePermissions(tempBaseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(tempBaseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERUSER, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + Path resolvedPath = new File(directory.getPath()).toPath(); + + Files.createDirectories(resolvedPath); + Files.setPosixFilePermissions(resolvedPath, PosixFilePermissions.fromString("rwxr-xr-x")); + + directory.ensureDirectory(); + } finally { + deleteRecursively(tempBaseDir); + } + } + + @Test + public void testResolvePerUserDirectoryRequiresExistingBaseDirectory() throws Exception { + Path tempDir = Files.createTempDirectory("auditSpoolDirParent"); + + try { + Path baseDir = tempDir.resolve("audit-spool"); + AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(baseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERUSER, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + + try { + directory.ensureDirectory(); + fail("Expected missing base audit spool directory to be rejected"); + } catch (IOException exception) { + assertTrue(exception.getMessage().contains("Base audit spool directory does not exist")); + } + + assertFalse(Files.exists(baseDir)); + } finally { + deleteRecursively(tempDir); + } + } + + @Test + public void testResolvePerUserDirectoryRejectsUnsafeBasePermissions() throws Exception { + Path tempBaseDir = Files.createTempDirectory("auditSpoolDir"); + + try { + AuditFileUtil.ResolvedDirectory directory = AuditFileUtil.resolveDirectory(tempBaseDir.toString(), + AuditFileUtil.SUBDIR_MODE_PERUSER, + AuditFileUtil.parsePermissions("755"), + AuditFileUtil.parsePermissions("644")); + Path resolvedPath = new File(directory.getPath()).toPath(); + + try { + directory.ensureDirectory(); + fail("Expected unsafe base audit spool permissions to be rejected"); + } catch (IOException exception) { + assertTrue(exception.getMessage().contains("Unsafe permissions on base audit spool directory")); + } + + assertFalse(Files.exists(resolvedPath)); + } finally { + deleteRecursively(tempBaseDir); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testResolveDirectoryRejectsUnknownMode() { + AuditFileUtil.resolveDirectory("/tmp/ranger-audit", "unknown", AuditFileUtil.parsePermissions("755"), AuditFileUtil.parsePermissions("644")); + } + + private static void deleteRecursively(Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return; + } + + try (java.util.stream.Stream paths = Files.walk(path)) { + paths.sorted(Comparator.reverseOrder()) + .forEach(currentPath -> { + try { + Files.deleteIfExists(currentPath); + } catch (IOException ignored) { + } + }); + } + } + } diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClientAccessDeniedException.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClientAccessDeniedException.java new file mode 100644 index 00000000000..351c0452cf0 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClientAccessDeniedException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.admin.client; + +public class RangerAdminClientAccessDeniedException extends Exception { + private final int httpStatus; + + public RangerAdminClientAccessDeniedException(int httpStatus, String message) { + super(message); + this.httpStatus = httpStatus; + } + + public int getHttpStatus() { + return httpStatus; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index bae3d295a90..b6cc7937e44 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -22,12 +22,12 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.sun.jersey.api.client.ClientResponse; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Cookie; import javax.ws.rs.core.NewCookie; @@ -150,17 +150,16 @@ public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, checkAndResetSessionCookie(response); - if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { - if (response == null) { - LOG.error("Error getting policies; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); - } else { - RESTResponse resp = RESTResponse.fromClientResponse(response); - if (LOG.isDebugEnabled()) { - LOG.debug("No change in policies. secureMode=" + isSecureMode + ", user=" + user - + ", response=" + resp + ", serviceName=" + serviceName - + ", " + "lastKnownVersion=" + lastKnownVersion - + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); - } + if (response == null) { + LOG.error("Error getting policies; Received NULL response!!. secureMode={}, user={}, serviceName={}", isSecureMode, user, serviceName); + throw new IOException("Error getting policies; received null response for serviceName=" + serviceName); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in policies. secureMode=" + isSecureMode + ", user=" + user + + ", response=" + resp + ", serviceName=" + serviceName + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); } ret = null; } else if (response.getStatus() == HttpServletResponse.SC_OK) { @@ -176,10 +175,14 @@ public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else if (isAccessDenied(response)) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + throw new RangerAdminClientAccessDeniedException(response.getStatus(), resp.getMessage()); } else { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.warn("Error getting policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); - ret = null; + throw new IOException("Error getting policies. response=" + resp + ", serviceName=" + serviceName); } if (LOG.isDebugEnabled()) { @@ -215,17 +218,19 @@ public RangerRoles getRolesIfUpdated(final long lastKnownRoleVersion, final long checkAndResetSessionCookie(response); - if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { - if (response == null) { - LOG.error("Error getting Roles; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); - } else { - RESTResponse resp = RESTResponse.fromClientResponse(response); - if (LOG.isDebugEnabled()) { - LOG.debug("No change in Roles. secureMode=" + isSecureMode + ", user=" + user - + ", response=" + resp + ", serviceName=" + serviceName - + ", " + "lastKnownRoleVersion=" + lastKnownRoleVersion - + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); - } + if (response == null) { + LOG.error("Error getting Roles; Received NULL response!!. secureMode={}, user={}, serviceName={}", isSecureMode, user, serviceName); + throw new IOException("Error getting Roles; received null response for serviceName=" + serviceName); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED || response.getStatus() == HttpServletResponse.SC_NO_CONTENT) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in Roles. secureMode={}, user={}, response={}, serviceName={}, lastKnownRoleVersion={}, lastActivationTimeInMillis={}", + isSecureMode, + user, + resp, + serviceName, + lastKnownRoleVersion, + lastActivationTimeInMillis); } ret = null; } else if (response.getStatus() == HttpServletResponse.SC_OK) { @@ -241,10 +246,14 @@ public RangerRoles getRolesIfUpdated(final long lastKnownRoleVersion, final long RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else if (isAccessDenied(response)) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting Roles. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + throw new RangerAdminClientAccessDeniedException(response.getStatus(), resp.getMessage()); } else { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.warn("Error getting Roles. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); - ret = null; + throw new IOException("Error getting Roles. response=" + resp + ", serviceName=" + serviceName); } if(LOG.isDebugEnabled()) { @@ -790,17 +799,19 @@ public ServiceTags getServiceTagsIfUpdated(final long lastKnownVersion, final lo checkAndResetSessionCookie(response); - if (response == null || response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED) { - if (response == null) { - LOG.error("Error getting tags; Received NULL response!!. secureMode=" + isSecureMode + ", user=" + user + ", serviceName=" + serviceName); - } else { - RESTResponse resp = RESTResponse.fromClientResponse(response); - if (LOG.isDebugEnabled()) { - LOG.debug("No change in tags. secureMode=" + isSecureMode + ", user=" + user - + ", response=" + resp + ", serviceName=" + serviceName - + ", " + "lastKnownVersion=" + lastKnownVersion - + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); - } + if (response == null) { + LOG.error("Error getting tags; Received NULL response!!. secureMode={}, user={}, serviceName={}", isSecureMode, user, serviceName); + throw new IOException("Error getting tags; received null response for serviceName=" + serviceName); + } else if (response.getStatus() == HttpServletResponse.SC_NOT_MODIFIED) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + if (LOG.isDebugEnabled()) { + LOG.debug("No change in tags. secureMode={}, user={}, response={}, serviceName={}, lastKnownVersion={}, lastActivationTimeInMillis={}", + isSecureMode, + user, + resp, + serviceName, + lastKnownVersion, + lastActivationTimeInMillis); } ret = null; } else if (response.getStatus() == HttpServletResponse.SC_OK) { @@ -815,10 +826,14 @@ public ServiceTags getServiceTagsIfUpdated(final long lastKnownVersion, final lo String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null; RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg); LOG.warn("Received 404 error code with body:[" + exceptionMsg + "], Ignoring"); + } else if (isAccessDenied(response)) { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.warn("Error getting tags. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); + throw new RangerAdminClientAccessDeniedException(response.getStatus(), resp.getMessage()); } else { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.warn("Error getting tags. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp + ", serviceName=" + serviceName); - ret = null; + throw new IOException("Error getting tags. response=" + resp + ", serviceName=" + serviceName); } if(LOG.isDebugEnabled()) { @@ -1155,6 +1170,12 @@ private ClientResponse getWithKerberos(String relativeURL, Map q }); } + private boolean isAccessDenied(ClientResponse response) { + int status = response == null ? 0 : response.getStatus(); + + return status == HttpServletResponse.SC_UNAUTHORIZED || status == HttpServletResponse.SC_FORBIDDEN; + } + private void checkAndResetSessionCookie(ClientResponse response) { if (isRangerCookieEnabled) { if (response == null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java index a926349200c..49fd48953cd 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.ranger.admin.client.RangerAdminClient; +import org.apache.ranger.admin.client.RangerAdminClientAccessDeniedException; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.util.ServiceTags; @@ -65,9 +66,12 @@ public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeIn } catch (ClosedByInterruptException closedByInterruptException) { LOG.error("Tag-retriever thread was interrupted while blocked on I/O"); throw new InterruptedException(); + } catch (RangerAdminClientAccessDeniedException ade) { + LOG.warn("Tag-retriever encountered authorization denial", ade); + throw ade; } catch (Exception e) { - LOG.error("Tag-retriever encounterd exception, exception=", e); - LOG.error("Returning null service tags"); + LOG.error("Tag-retriever encountered exception", e); + throw e; } } @@ -75,4 +79,3 @@ public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeIn } } - diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 74e6fdae376..982bc5e18be 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -22,6 +22,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.admin.client.RangerAdminClientAccessDeniedException; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.model.RangerPolicy; @@ -109,7 +110,7 @@ public void init() { serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); String cacheFilePermsString = StringUtils.trim(getConfig(propertyPrefix + ".policy.cache.file.perms", "644")); - this.cacheFilePerms = FileUtils.parsePermissions(cacheFilePermsString); + Set configuredCacheFilePerms = FileUtils.parsePermissions(cacheFilePermsString); if (StringUtils.isNotBlank(tagRetrieverClassName)) { @@ -132,12 +133,18 @@ public void init() { if (tagRetriever != null) { disableCacheIfServiceNotFound = getBooleanConfig(propertyPrefix + ".disable.cache.if.servicenotfound", true); String cacheDir = getConfig(propertyPrefix + ".policy.cache.dir", null); + String cacheDirPermsString = StringUtils.trim(getConfig(propertyPrefix + ".policy.cache.dir.perms", "755")); + RangerLocalDirectory.ResolvedDirectory cacheDirectory = RangerLocalDirectory.resolve(cacheDir, + getConfig(propertyPrefix + ".policy.cache.subdir.mode", RangerLocalDirectory.SUBDIR_MODE_DISABLED), + FileUtils.parsePermissions(cacheDirPermsString), + configuredCacheFilePerms); String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName); cacheFilename = cacheFilename.replace(File.separatorChar, '_'); cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); - String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename); + this.cacheFilePerms = cacheDirectory.getFilePermissions(); + String cacheFile = cacheDirectory.getPath() == null ? null : (cacheDirectory.getPath() + File.separator + cacheFilename); createLock(); @@ -148,7 +155,7 @@ public void init() { tagRetriever.setPluginContext(getPluginContext()); tagRetriever.init(enricherDef.getEnricherOptions()); - tagRefresher = new RangerTagRefresher(tagRetriever, this, -1L, tagDownloadQueue, cacheFile, cacheFilePerms); + tagRefresher = new RangerTagRefresher(tagRetriever, this, -1L, tagDownloadQueue, cacheDirectory, cacheFile, cacheFilePerms); LOG.info("Created RangerTagRefresher Thread(" + tagRefresher.getName() + ")"); try { @@ -885,14 +892,16 @@ static class RangerTagRefresher extends Thread { private final BlockingQueue tagDownloadQueue; private long lastActivationTimeInMillis; private final Set filePermissions; + private final RangerLocalDirectory.ResolvedDirectory cacheDirectory; private final String cacheFile; private boolean hasProvidedTagsToReceiver; - RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long lastKnownVersion, BlockingQueue tagDownloadQueue, String cacheFile, Set filePermissions) { + RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long lastKnownVersion, BlockingQueue tagDownloadQueue, RangerLocalDirectory.ResolvedDirectory cacheDirectory, String cacheFile, Set filePermissions) { this.tagRetriever = tagRetriever; this.tagEnricher = tagEnricher; this.lastKnownVersion = lastKnownVersion; this.tagDownloadQueue = tagDownloadQueue; + this.cacheDirectory = cacheDirectory; this.cacheFile = cacheFile; this.filePermissions = filePermissions; setName("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ")-" + getId()); @@ -949,6 +958,7 @@ private void populateTags() throws InterruptedException { try { serviceTags = tagRetriever.retrieveTags(lastKnownVersion, lastActivationTimeInMillis); + tagEnricher.getPluginContext().setTagDownloadAuthzDenied(false); if (serviceTags == null) { if (!hasProvidedTagsToReceiver) { @@ -983,10 +993,23 @@ private void populateTags() throws InterruptedException { setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = -1L; } + } catch (RangerAdminClientAccessDeniedException ade) { + LOG.warn("RangerTagRefresher(serviceName={}).populateTags(): tag refresh authorization denied. Access checks will fail closed if configured.", tagRetriever.getServiceName(), ade); + tagEnricher.getPluginContext().setTagDownloadAuthzDenied(true); } catch (InterruptedException interruptedException) { throw interruptedException; } catch (Exception e) { - LOG.error("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags(): Encountered unexpected exception. Ignoring", e); + LOG.error("RangerTagRefresher(serviceName={}).populateTags(): Encountered unexpected exception. Ignoring", tagRetriever.getServiceName(), e); + if (!hasProvidedTagsToReceiver) { + ServiceTags cachedServiceTags = loadFromCache(); + + if (cachedServiceTags != null) { + tagEnricher.setServiceTags(cachedServiceTags); + hasProvidedTagsToReceiver = true; + lastKnownVersion = cachedServiceTags.getTagVersion() == null ? -1L : cachedServiceTags.getTagVersion(); + setLastActivationTimeInMillis(System.currentTimeMillis()); + } + } } } else { @@ -1051,6 +1074,13 @@ final ServiceTags loadFromCache() { File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile); + try { + cacheDirectory.ensureDirectory(); + } catch (Exception excp) { + LOG.error("failed to validate service-tags cache directory", excp); + return null; + } + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { Reader reader = null; @@ -1098,6 +1128,7 @@ final void saveToCache(ServiceTags serviceTags) { Writer writer = null; try { + cacheDirectory.ensureDirectory(); if (!cacheFile.exists()) { Files.createFile(cacheFile.toPath(), PosixFilePermissions.asFileAttribute(this.filePermissions)); Files.setPosixFilePermissions(cacheFile.toPath(), this.filePermissions); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java index 5dc25245c9e..2ba963a2ba1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java @@ -41,6 +41,9 @@ public class RangerPluginContext { private RangerAuthContext authContext; private RangerAuthContextListener authContextListener; private RangerAdminClient adminClient; + private volatile boolean policyDownloadAuthzDenied; + private volatile boolean roleDownloadAuthzDenied; + private volatile boolean tagDownloadAuthzDenied; private final Map> resourceMatchers = new HashMap<>(); private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); // fair lock @@ -124,6 +127,16 @@ void cleanResourceMatchers() { public void setAuthContextListener(RangerAuthContextListener authContextListener) { this.authContextListener = authContextListener; } + public boolean isPolicyRefreshAuthzDenied() { + return policyDownloadAuthzDenied || roleDownloadAuthzDenied || tagDownloadAuthzDenied; + } + + public void setPolicyDownloadAuthzDenied(boolean policyDownloadAuthzDenied) { this.policyDownloadAuthzDenied = policyDownloadAuthzDenied; } + + public void setRoleDownloadAuthzDenied(boolean roleDownloadAuthzDenied) { this.roleDownloadAuthzDenied = roleDownloadAuthzDenied; } + + public void setTagDownloadAuthzDenied(boolean tagDownloadAuthzDenied) { this.tagDownloadAuthzDenied = tagDownloadAuthzDenied; } + public void notifyAuthContextChanged() { RangerAuthContextListener authContextListener = this.authContextListener; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index cfa9db1621a..698d9a478a8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -68,6 +68,7 @@ public class RangerBasePlugin { private final Map logHistoryList = new Hashtable<>(); private final int logInterval = 30000; // 30 seconds private final DownloadTrigger accessTrigger = new DownloadTrigger(); + private final boolean failClosedOnPolicyRefreshAuthzDenied; private PolicyRefresher refresher; private RangerPolicyEngine policyEngine; private RangerAuthContext currentAuthContext; @@ -114,6 +115,9 @@ public RangerBasePlugin(RangerPluginConfig pluginConfig) { setIsFallbackSupported(pluginConfig.getBoolean(pluginConfig.getPropertyPrefix() + ".is.fallback.supported", false)); setServiceAdmins(serviceAdmins); + String authzDeniedMode = pluginConfig.get(pluginConfig.getPropertyPrefix() + ".policy.refresh.authz.denied.mode", "continue"); + this.failClosedOnPolicyRefreshAuthzDenied = StringUtils.equals(authzDeniedMode, "failclosed"); + String ugiPrefix = pluginConfig.getPropertyPrefix() + ".ugi"; boolean initUgi = pluginConfig.getBoolean(ugiPrefix + ".initialize", false); @@ -549,6 +553,10 @@ public RangerAccessResultProcessor getResultProcessor() { return this.resultProcessor; } + public boolean isPolicyRefreshAuthzDenied() { + return failClosedOnPolicyRefreshAuthzDenied && pluginContext.isPolicyRefreshAuthzDenied(); + } + public RangerAccessResult isAccessAllowed(RangerAccessRequest request) { return isAccessAllowed(request, resultProcessor); } @@ -560,12 +568,15 @@ public Collection isAccessAllowed(Collection isAccessAllowed(Collection requests, RangerAccessResultProcessor resultProcessor) { Collection ret = null; RangerPolicyEngine policyEngine = this.policyEngine; + boolean isPolicyRefreshAuthzDenied = isPolicyRefreshAuthzDenied(); - if (policyEngine != null) { + if (isPolicyRefreshAuthzDenied) { + ret = new ArrayList<>(); + for (RangerAccessRequest request : requests) { + ret.add(getPolicyRefreshAuthzDeniedResult(request, RangerPolicy.POLICY_TYPE_ACCESS)); + } + } else if (policyEngine != null) { ret = policyEngine.evaluatePolicies(requests, RangerPolicy.POLICY_TYPE_ACCESS, null); } - if (CollectionUtils.isNotEmpty(ret)) { + if (!isPolicyRefreshAuthzDenied && CollectionUtils.isNotEmpty(ret)) { for (RangerChainedPlugin chainedPlugin : chainedPlugins) { Collection chainedResults = chainedPlugin.isAccessAllowed(requests); @@ -646,8 +663,11 @@ public Collection isAccessAllowed(Collection cacheFilePerms; private final Set cacheDirPerms; private final BlockingQueue policyDownloadQueue = new LinkedBlockingQueue<>(); @@ -73,12 +73,17 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.plugIn = plugIn; this.serviceType = plugIn.getServiceType(); this.serviceName = plugIn.getServiceName(); - this.cacheDir = pluginConfig.get(propertyPrefix + ".policy.cache.dir"); String cacheFilePermsString = StringUtils.defaultIfEmpty(StringUtils.trim(pluginConfig.get(propertyPrefix + ".policy.cache.file.perms")), "644"); - this.cacheFilePerms = FileUtils.parsePermissions(cacheFilePermsString); - String cacheDirPermsString = StringUtils.defaultIfEmpty(StringUtils.trim(pluginConfig.get(propertyPrefix + ".policy.cache.dir.perms")), "755"); - this.cacheDirPerms = FileUtils.parsePermissions(cacheDirPermsString); + String baseCacheDir = pluginConfig.get(propertyPrefix + ".policy.cache.dir"); + this.cacheDirectory = RangerLocalDirectory.resolve(baseCacheDir, + pluginConfig.get(propertyPrefix + ".policy.cache.subdir.mode", RangerLocalDirectory.SUBDIR_MODE_DISABLED), + FileUtils.parsePermissions(cacheDirPermsString), + FileUtils.parsePermissions(cacheFilePermsString)); + + this.cacheDir = this.cacheDirectory.getPath(); + this.cacheFilePerms = this.cacheDirectory.getFilePermissions(); + this.cacheDirPerms = this.cacheDirectory.getDirPermissions(); String appId = StringUtils.isEmpty(plugIn.getAppId()) ? serviceType : plugIn.getAppId(); String cacheFilename = String.format("%s_%s.json", appId, serviceName); @@ -91,7 +96,7 @@ public PolicyRefresher(RangerBasePlugin plugIn) { RangerPluginContext pluginContext = plugIn.getPluginContext(); RangerAdminClient adminClient = pluginContext.getAdminClient(); this.rangerAdmin = (adminClient != null) ? adminClient : pluginContext.createAdminClient(pluginConfig); - this.rolesProvider = new RangerRolesProvider(getServiceType(), appId, getServiceName(), rangerAdmin, cacheDir, pluginConfig); + this.rolesProvider = new RangerRolesProvider(getServiceType(), appId, getServiceName(), rangerAdmin, baseCacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000L); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); @@ -240,8 +245,8 @@ private void loadPolicy() { } try { - //load policy from PolicyAdmin ServicePolicies svcPolicies = loadPolicyfromPolicyAdmin(); + plugIn.getPluginContext().setPolicyDownloadAuthzDenied(false); if (svcPolicies == null) { //if Policy fetch from Policy Admin Fails, load from cache @@ -268,6 +273,9 @@ private void loadPolicy() { serviceDefSetInPlugin = true; } } + } catch (RangerAdminClientAccessDeniedException ade) { + plugIn.getPluginContext().setPolicyDownloadAuthzDenied(true); + LOG.warn("PolicyRefresher(serviceName={}): policy refresh authorization denied. Access checks will fail closed if configured.", serviceName, ade); } catch (RangerServiceNotFoundException snfe) { if (!serviceDefSetInPlugin) { disableCache(); @@ -278,6 +286,20 @@ private void loadPolicy() { } } catch (Exception excp) { LOG.error("Encountered unexpected exception, ignoring..", excp); + if (!policiesSetInPlugin) { + ServicePolicies svcPolicies = loadFromCache(); + + if (svcPolicies != null) { + plugIn.setPolicies(svcPolicies); + policiesSetInPlugin = true; + serviceDefSetInPlugin = false; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; + } else if (!serviceDefSetInPlugin) { + plugIn.setPolicies(null); + serviceDefSetInPlugin = true; + } + } } RangerPerfTracer.log(perf); @@ -287,7 +309,7 @@ private void loadPolicy() { } } - private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFoundException { + private ServicePolicies loadPolicyfromPolicyAdmin() throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin()"); @@ -323,11 +345,14 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound } } } catch (RangerServiceNotFoundException snfe) { - LOG.error("PolicyRefresher(serviceName=" + serviceName + "): failed to find service. Will clean up local cache of policies (" + lastKnownVersion + ")", snfe); + LOG.error("PolicyRefresher(serviceName={}): failed to find service. Will clean up local cache of policies ({})", serviceName, lastKnownVersion, snfe); throw snfe; + } catch (RangerAdminClientAccessDeniedException ade) { + LOG.warn("PolicyRefresher(serviceName={}): failed to refresh policies due to authorization denial ({})", serviceName, lastKnownVersion, ade); + throw ade; } catch (Exception excp) { - LOG.error("PolicyRefresher(serviceName=" + serviceName + "): failed to refresh policies. Will continue to use last known version of policies (" + lastKnownVersion + ")", excp); - svcPolicies = null; + LOG.error("PolicyRefresher(serviceName={}): failed to refresh policies. Will continue to use last known version of policies ({})", serviceName, lastKnownVersion, excp); + throw excp; } RangerPerfTracer.log(perf); @@ -350,6 +375,13 @@ private ServicePolicies loadFromCache() { File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + try { + cacheDirectory.ensureDirectory(); + } catch (Exception excp) { + LOG.error("failed to validate cache directory {}", cacheDir, excp); + return null; + } + if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { Reader reader = null; @@ -411,17 +443,15 @@ public void saveToCache(ServicePolicies policies) { String backupCacheFileName = cacheFileName + "_" + policies.getPolicyVersion(); String realCacheFileName = CollectionUtils.isNotEmpty(policies.getPolicyDeltas()) ? backupCacheFileName : cacheFileName; - // Create the cacheDir if it doesn't already exist File cacheDirTmp = new File(realCacheDirName); - if (cacheDirTmp.exists()) { - cacheFile = new File(realCacheDirName + File.separator + realCacheFileName); - } else { - try { + try { + cacheDirectory.ensureDirectory(); + if (!cacheDirTmp.exists()) { FileUtils.createDirectoryWithPermissions(cacheDirTmp, cacheDirPerms); - cacheFile = new File(realCacheDirName + File.separator + realCacheFileName); - } catch (Exception ex) { - LOG.error("Cannot create cache directory", ex); } + cacheFile = new File(realCacheDirName + File.separator + realCacheFileName); + } catch (Exception ex) { + LOG.error("Cannot create cache directory", ex); } if (CollectionUtils.isEmpty(policies.getPolicyDeltas())) { backupCacheFile = new File(realCacheDirName + File.separator + backupCacheFileName); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerLocalDirectory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerLocalDirectory.java new file mode 100644 index 00000000000..8ada3d536cc --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerLocalDirectory.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.ranger.audit.utils.LocalDirectoryResolver; + +import java.io.IOException; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Set; + +public final class RangerLocalDirectory { + public static final String SUBDIR_MODE_DISABLED = "disabled"; + public static final String SUBDIR_MODE_PERUSER = "peruser"; + public static final String SUBDIR_MODE_PERGROUP = "pergroup"; + + private RangerLocalDirectory() { + } + + public static ResolvedDirectory resolve(String baseDir, String subdirMode, Set defaultDirPerms, Set defaultFilePerms) { + return new ResolvedDirectory(LocalDirectoryResolver.resolveLocalDirectory(baseDir, subdirMode, defaultDirPerms, defaultFilePerms)); + } + + public static final class ResolvedDirectory { + private final LocalDirectoryResolver.ResolvedDirectory delegate; + + private ResolvedDirectory(LocalDirectoryResolver.ResolvedDirectory delegate) { + this.delegate = delegate; + } + + public String getPath() { + return delegate.getPath(); + } + + public Set getDirPermissions() { + return delegate.getDirPermissions(); + } + + public Set getFilePermissions() { + return delegate.getFilePermissions(); + } + + public void ensureDirectory() throws IOException { + delegate.ensureDirectory(); + } + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java index 10f448143e2..0a47e0d4c45 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.ranger.admin.client.RangerAdminClient; +import org.apache.ranger.admin.client.RangerAdminClientAccessDeniedException; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.service.RangerBasePlugin; @@ -52,8 +53,8 @@ public class RangerRolesProvider { private final String cacheFileName; private final String cacheFileNamePrefix; private final String cacheDir; + private final RangerLocalDirectory.ResolvedDirectory cacheDirectory; private final Set cacheFilePerms; - private final Set cacheDirPerms; private final boolean disableCacheIfServiceNotFound; private long lastActivationTimeInMillis; @@ -80,15 +81,17 @@ public RangerRolesProvider(String serviceType, String appId, String serviceName, cacheFilename = cacheFilename.replace(File.separatorChar, '_'); cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_'); - this.cacheFileName = cacheFilename; - this.cacheDir = cacheDir; String propertyPrefix = config.getPropertyPrefix(); disableCacheIfServiceNotFound = config.getBoolean(propertyPrefix + ".disable.cache.if.servicenotfound", true); String cacheFilePermsString = StringUtils.defaultIfEmpty(StringUtils.trim(config.get(propertyPrefix + ".policy.cache.file.perms")), "644"); - this.cacheFilePerms = FileUtils.parsePermissions(cacheFilePermsString); - String cacheDirPermsString = StringUtils.defaultIfEmpty(StringUtils.trim(config.get(propertyPrefix + ".policy.cache.dir.perms")), "755"); - this.cacheDirPerms = FileUtils.parsePermissions(cacheDirPermsString); + this.cacheDirectory = RangerLocalDirectory.resolve(cacheDir, + config.get(propertyPrefix + ".policy.cache.subdir.mode", RangerLocalDirectory.SUBDIR_MODE_DISABLED), + FileUtils.parsePermissions(cacheDirPermsString), + FileUtils.parsePermissions(cacheFilePermsString)); + this.cacheFileName = cacheFilename; + this.cacheDir = this.cacheDirectory.getPath(); + this.cacheFilePerms = this.cacheDirectory.getFilePermissions(); if (LOG.isDebugEnabled()) { LOG.debug("<== RangerRolesProvider(serviceName=" + serviceName + ").RangerRolesProvider()"); @@ -119,8 +122,8 @@ public void loadUserGroupRoles(RangerBasePlugin plugIn) { } try { - //load userGroupRoles from ranger admin RangerRoles roles = loadUserGroupRolesFromAdmin(); + plugIn.getPluginContext().setRoleDownloadAuthzDenied(false); if (roles == null) { //if userGroupRoles fetch from ranger Admin Fails, load from cache @@ -146,6 +149,9 @@ public void loadUserGroupRoles(RangerBasePlugin plugIn) { serviceDefSetInPlugin = true; } } + } catch (RangerAdminClientAccessDeniedException ade) { + plugIn.getPluginContext().setRoleDownloadAuthzDenied(true); + LOG.warn("RangerRolesProvider(serviceName={}): role refresh authorization denied. Access checks will fail closed if configured.", serviceName, ade); } catch (RangerServiceNotFoundException snfe) { if (disableCacheIfServiceNotFound) { disableCache(); @@ -156,6 +162,19 @@ public void loadUserGroupRoles(RangerBasePlugin plugIn) { } } catch (Exception excp) { LOG.error("Encountered unexpected exception, ignoring..", excp); + if (!rangerUserGroupRolesSetInPlugin) { + RangerRoles roles = loadUserGroupRolesFromCache(); + + if (roles != null) { + plugIn.setRoles(roles); + rangerUserGroupRolesSetInPlugin = true; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownRoleVersion = roles.getRoleVersion() != null ? roles.getRoleVersion() : -1; + } else if (!serviceDefSetInPlugin) { + plugIn.setRoles(null); + serviceDefSetInPlugin = true; + } + } } RangerPerfTracer.log(perf); @@ -165,7 +184,7 @@ public void loadUserGroupRoles(RangerBasePlugin plugIn) { } } - private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundException { + private RangerRoles loadUserGroupRolesFromAdmin() throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerRolesProvider(serviceName=" + serviceName + ").loadUserGroupRolesFromAdmin()"); @@ -196,9 +215,12 @@ private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundEx } catch (RangerServiceNotFoundException snfe) { LOG.error("RangerRolesProvider(serviceName=" + serviceName + "): failed to find service. Will clean up local cache of roles (" + lastKnownRoleVersion + ")", snfe); throw snfe; + } catch (RangerAdminClientAccessDeniedException ade) { + LOG.warn("RangerRolesProvider(serviceName={}): failed to refresh roles due to authorization denial (lastKnownRoleVersion={})", serviceName, lastKnownRoleVersion, ade); + throw ade; } catch (Exception excp) { - LOG.error("RangerRolesProvider(serviceName=" + serviceName + "): failed to refresh roles. Will continue to use last known version of roles (" + "lastKnowRoleVersion= " + lastKnownRoleVersion, excp); - roles = null; + LOG.error("RangerRolesProvider(serviceName={}): failed to refresh roles. Will continue to use last known version of roles (lastKnowRoleVersion= {}", serviceName, lastKnownRoleVersion, excp); + throw excp; } RangerPerfTracer.log(perf); @@ -220,6 +242,13 @@ private RangerRoles loadUserGroupRolesFromCache() { File cacheFile = cacheDir == null ? null : new File(cacheDir + File.separator + cacheFileName); + try { + cacheDirectory.ensureDirectory(); + } catch (Exception excp) { + LOG.error("failed to validate roles cache directory {}", cacheDir, excp); + return null; + } + if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { Reader reader = null; @@ -280,17 +309,11 @@ public void saveToCache(RangerRoles roles) { if(roles != null) { File cacheFile = null; if (cacheDir != null) { - // Create the cacheDir if it doesn't already exist - File cacheDirTmp = new File(cacheDir); - if (cacheDirTmp.exists()) { + try { + cacheDirectory.ensureDirectory(); cacheFile = new File(cacheDir + File.separator + cacheFileName); - } else { - try { - FileUtils.createDirectoryWithPermissions(cacheDirTmp, cacheDirPerms); - cacheFile = new File(cacheDir + File.separator + cacheFileName); - } catch (Exception ex) { - LOG.error("Cannot create cache directory", ex); - } + } catch (Exception ex) { + LOG.error("Cannot create cache directory", ex); } } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-nestedstructure.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-nestedstructure.json index 8aa1b705bdc..dd82ddcb445 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-nestedstructure.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-nestedstructure.json @@ -9,7 +9,8 @@ }, "configs": [ { "itemId": 1, "name": "commonNameForCertificate", "type": "string", "mandatory": false }, - { "itemId": 2, "name": "policy.download.auth.users", "type": "string", "mandatory": false } + { "itemId": 2, "name": "policy.download.auth.users", "type": "string", "mandatory": false }, + { "itemId": 3, "name": "policy.download.auth.groups", "type": "string", "mandatory": false } ], "resources": [ { @@ -183,4 +184,4 @@ } ] } -} \ No newline at end of file +} diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java index cac2787e82d..9681d72a09c 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java @@ -60,6 +60,35 @@ public void testBasePluginHive() { runTestsFromResourceFile("/plugin/test_base_plugin_hive.json"); } + @Test + public void testPolicyRefreshAuthzDeniedFailsClosedWhenConfigured() { + RangerBasePluginTestCase testCase = readTestCase(new InputStreamReader(this.getClass().getResourceAsStream("/plugin/test_base_plugin_hive.json"))); + TestData test = testCase.tests.get(0); + RangerPluginConfig config = new RangerPluginConfig(testCase.policies.getServiceDef().getName(), testCase.policies.getServiceName(), "hive", "cl1", "on-prem", peOptions); + + config.set(config.getPropertyPrefix() + ".policy.refresh.authz.denied.mode", "failclosed"); + + RangerBasePlugin plugin = new RangerBasePlugin(config, testCase.policies, testCase.tags, testCase.roles, testCase.userStore); + + RangerAccessResult allowedResult = plugin.isAccessAllowed(test.request); + assertNotNull(allowedResult); + assertTrue(allowedResult.getIsAllowed()); + + plugin.getPluginContext().setPolicyDownloadAuthzDenied(true); + + RangerAccessResult deniedResult = plugin.isAccessAllowed(test.request); + assertNotNull(deniedResult); + assertFalse(deniedResult.getIsAllowed()); + assertTrue(deniedResult.getIsAccessDetermined()); + assertEquals("Policy refresh authorization denied by Ranger Admin", deniedResult.getReason()); + + plugin.getPluginContext().setPolicyDownloadAuthzDenied(false); + + RangerAccessResult restoredResult = plugin.isAccessAllowed(test.request); + assertNotNull(restoredResult); + assertTrue(restoredResult.getIsAllowed()); + } + private void runTestsFromResourceFile(String resourceFile) { InputStream inStream = this.getClass().getResourceAsStream(resourceFile); InputStreamReader reader = new InputStreamReader(inStream); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/util/RangerLocalDirectoryTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/util/RangerLocalDirectoryTest.java new file mode 100644 index 00000000000..4d1f69d67e4 --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/util/RangerLocalDirectoryTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Comparator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class RangerLocalDirectoryTest { + @Test + public void testPerUserDirectoryUsesPrivatePermissions() throws Exception { + Path baseDir = Files.createTempDirectory("ranger-policy-cache"); + + try { + Files.setPosixFilePermissions(baseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + RangerLocalDirectory.ResolvedDirectory directory = RangerLocalDirectory.resolve(baseDir.toString(), + RangerLocalDirectory.SUBDIR_MODE_PERUSER, + FileUtils.parsePermissions("755"), + FileUtils.parsePermissions("644")); + + directory.ensureDirectory(); + + Path resolvedPath = Paths.get(directory.getPath()); + assertTrue(Files.isDirectory(resolvedPath)); + assertEquals(PosixFilePermissions.fromString("rwxr-xr-x"), Files.getPosixFilePermissions(baseDir)); + assertEquals(PosixFilePermissions.fromString("rwx------"), Files.getPosixFilePermissions(resolvedPath)); + assertEquals(PosixFilePermissions.fromString("rw-------"), directory.getFilePermissions()); + } finally { + deleteRecursively(baseDir); + } + } + + @Test + public void testPerGroupDirectoryUsesGroupPermissions() throws Exception { + Path baseDir = Files.createTempDirectory("ranger-policy-cache"); + + try { + Files.setPosixFilePermissions(baseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + RangerLocalDirectory.ResolvedDirectory directory = RangerLocalDirectory.resolve(baseDir.toString(), + RangerLocalDirectory.SUBDIR_MODE_PERGROUP, + FileUtils.parsePermissions("755"), + FileUtils.parsePermissions("644")); + + directory.ensureDirectory(); + + Path resolvedPath = Paths.get(directory.getPath()); + assertTrue(Files.isDirectory(resolvedPath)); + assertEquals(PosixFilePermissions.fromString("rwxr-xr-x"), Files.getPosixFilePermissions(baseDir)); + assertEquals(PosixFilePermissions.fromString("rwxrwx---"), Files.getPosixFilePermissions(resolvedPath)); + assertEquals(PosixFilePermissions.fromString("rw-rw----"), directory.getFilePermissions()); + } finally { + deleteRecursively(baseDir); + } + } + + @Test(expected = IOException.class) + public void testUnsafeExistingDirectoryPermissionsAreRejected() throws Exception { + Path baseDir = Files.createTempDirectory("ranger-policy-cache"); + + try { + Files.setPosixFilePermissions(baseDir, PosixFilePermissions.fromString("rwxr-xr-x")); + RangerLocalDirectory.ResolvedDirectory directory = RangerLocalDirectory.resolve(baseDir.toString(), + RangerLocalDirectory.SUBDIR_MODE_PERUSER, + FileUtils.parsePermissions("755"), + FileUtils.parsePermissions("644")); + Path resolvedPath = Paths.get(directory.getPath()); + + Files.createDirectories(resolvedPath); + Files.setPosixFilePermissions(resolvedPath, PosixFilePermissions.fromString("rwxr-xr-x")); + + directory.ensureDirectory(); + } finally { + deleteRecursively(baseDir); + } + } + + @Test + public void testPerUserDirectoryRequiresExistingBaseDirectory() throws Exception { + Path tempDir = Files.createTempDirectory("ranger-policy-cache-parent"); + + try { + Path baseDir = tempDir.resolve("policy-cache"); + RangerLocalDirectory.ResolvedDirectory directory = RangerLocalDirectory.resolve(baseDir.toString(), + RangerLocalDirectory.SUBDIR_MODE_PERUSER, + FileUtils.parsePermissions("755"), + FileUtils.parsePermissions("644")); + + try { + directory.ensureDirectory(); + fail("Expected missing base directory to be rejected"); + } catch (IOException exception) { + assertTrue(exception.getMessage().contains("Base local directory does not exist")); + } + + assertFalse(Files.exists(baseDir)); + } finally { + deleteRecursively(tempDir); + } + } + + @Test + public void testPerUserDirectoryRejectsUnsafeBasePermissions() throws Exception { + Path baseDir = Files.createTempDirectory("ranger-policy-cache"); + + try { + RangerLocalDirectory.ResolvedDirectory directory = RangerLocalDirectory.resolve(baseDir.toString(), + RangerLocalDirectory.SUBDIR_MODE_PERUSER, + FileUtils.parsePermissions("755"), + FileUtils.parsePermissions("644")); + Path resolvedPath = Paths.get(directory.getPath()); + + try { + directory.ensureDirectory(); + fail("Expected unsafe base directory permissions to be rejected"); + } catch (IOException exception) { + assertTrue(exception.getMessage().contains("Unsafe permissions on base local directory")); + } + + assertFalse(Files.exists(resolvedPath)); + } finally { + deleteRecursively(baseDir); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testUnknownSubdirModeIsRejected() { + RangerLocalDirectory.resolve("/tmp/ranger-cache", "unknown", FileUtils.parsePermissions("755"), FileUtils.parsePermissions("644")); + } + + private static void deleteRecursively(Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return; + } + + try (java.util.stream.Stream paths = Files.walk(path)) { + paths.sorted(Comparator.reverseOrder()).forEach(currentPath -> { + try { + Files.deleteIfExists(currentPath); + } catch (IOException ignored) { + } + }); + } + } +} diff --git a/plugin-nestedstructure/src/test/resources/servicedef-nestedstructure.json b/plugin-nestedstructure/src/test/resources/servicedef-nestedstructure.json index 9cf6ba85bbf..137f5d9b874 100644 --- a/plugin-nestedstructure/src/test/resources/servicedef-nestedstructure.json +++ b/plugin-nestedstructure/src/test/resources/servicedef-nestedstructure.json @@ -9,7 +9,8 @@ }, "configs": [ { "itemId": 1, "name": "commonNameForCertificate", "type": "string", "mandatory": false }, - { "itemId": 2, "name": "policy.download.auth.users", "type": "string", "mandatory": false } + { "itemId": 2, "name": "policy.download.auth.users", "type": "string", "mandatory": false }, + { "itemId": 3, "name": "policy.download.auth.groups", "type": "string", "mandatory": false } ], "resources": [ { @@ -183,4 +184,4 @@ } ] } -} \ No newline at end of file +} diff --git a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql index d06d7879775..dc0b45d730a 100644 --- a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql +++ b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql @@ -1904,4 +1904,5 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10062',UTC_TIMESTAMP(),'Ranger 2.5.0',UTC_TIMESTAMP(),'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10063',UTC_TIMESTAMP(),'Ranger 2.5.0',UTC_TIMESTAMP(),'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10064',UTC_TIMESTAMP(),'Ranger 2.6.0',UTC_TIMESTAMP(),'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10065',UTC_TIMESTAMP(),'Ranger 2.6.0',UTC_TIMESTAMP(),'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y'); diff --git a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql index 4ec797d5548..05594d345c2 100644 --- a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql +++ b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql @@ -2098,5 +2098,6 @@ INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,act INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval,'J10062',sys_extract_utc(systimestamp),'Ranger 2.5.0',sys_extract_utc(systimestamp),'localhost','Y'); INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval,'J10063',sys_extract_utc(systimestamp),'Ranger 2.5.0',sys_extract_utc(systimestamp),'localhost','Y'); INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval,'J10064',sys_extract_utc(systimestamp),'Ranger 2.6.0',sys_extract_utc(systimestamp),'localhost','Y'); +INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval,'J10065',sys_extract_utc(systimestamp),'Ranger 2.6.0',sys_extract_utc(systimestamp),'localhost','Y'); INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval,'JAVA_PATCHES',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y'); commit; diff --git a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql index ce1ebe53992..c196184df27 100644 --- a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql +++ b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql @@ -2047,4 +2047,5 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10062',current_timestamp,'Ranger 2.5.0',current_timestamp,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10063',current_timestamp,'Ranger 2.5.0',current_timestamp,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10064',current_timestamp,'Ranger 2.6.0',current_timestamp,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10065',current_timestamp,'Ranger 2.6.0',current_timestamp,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y'); diff --git a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql index f06a230eae6..776ebe9b2ec 100644 --- a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql +++ b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql @@ -2474,6 +2474,8 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active GO INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10064',CURRENT_TIMESTAMP,'Ranger 2.6.0',CURRENT_TIMESTAMP,'localhost','Y'); GO +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10065',CURRENT_TIMESTAMP,'Ranger 2.6.0',CURRENT_TIMESTAMP,'localhost','Y'); +GO INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y'); GO exit diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql index 9a2f0d655c1..8d7677ede40 100644 --- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql @@ -4266,5 +4266,6 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10062',CURRENT_TIMESTAMP,'Ranger 2.5.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10063',CURRENT_TIMESTAMP,'Ranger 2.5.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10064',CURRENT_TIMESTAMP,'Ranger 2.6.0',CURRENT_TIMESTAMP,'localhost','Y'); +INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('J10065',CURRENT_TIMESTAMP,'Ranger 2.6.0',CURRENT_TIMESTAMP,'localhost','Y'); INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('JAVA_PATCHES',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y'); GO diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index ddf7effc30e..23857e5654b 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1412,6 +1413,67 @@ public boolean isUserAllowed(RangerService rangerService, String cfgNameAllowedU return false; } + public boolean isUserInAllowedGroup(RangerService rangerService, String cfgNameAllowedGroups) { + String user = null; + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null) { + user = userSession.getLoginId(); + } + + if (StringUtils.isNotBlank(user)) { + return isAnyGroupInConfigParameter(rangerService, cfgNameAllowedGroups, getGroupsForLoginId(user)); + } + + return false; + } + + private Set getGroupsForLoginId(String loginId) { + Set groups = new LinkedHashSet<>(); + + for (String userName : getUserNameCandidates(loginId)) { + if (StringUtils.isBlank(userName)) { + continue; + } + + XXUser xUser = daoManager != null && daoManager.getXXUser() != null ? daoManager.getXXUser().findByUserName(userName) : null; + + if (xUser != null && daoManager.getXXGroup() != null) { + List userGroups = daoManager.getXXGroup().findByUserId(xUser.getId()); + + if (CollectionUtils.isNotEmpty(userGroups)) { + for (XXGroup userGroup : userGroups) { + if (userGroup != null && StringUtils.isNotBlank(userGroup.getName())) { + groups.add(userGroup.getName()); + } + } + } + } + + if (userMgr != null && userMgr.xUserMgr != null) { + Set xUserMgrGroups = userMgr.xUserMgr.getGroupsForUser(userName); + + if (CollectionUtils.isNotEmpty(xUserMgrGroups)) { + groups.addAll(xUserMgrGroups); + } + } + } + + return groups; + } + + private Set getUserNameCandidates(String loginId) { + Set ret = new LinkedHashSet<>(); + String userName = StringUtils.trim(loginId); + + if (StringUtils.isNotBlank(userName)) { + ret.add(userName); + ret.add(StringUtils.substringBefore(userName, "@")); + ret.add(StringUtils.substringBefore(StringUtils.substringBefore(userName, "@"), "/")); + } + + return ret; + } + public boolean isUserAllowedForGrantRevoke(RangerService rangerService, String userName) { return isUserInConfigParameter(rangerService, ServiceREST.Allowed_User_List_For_Grant_Revoke, userName); } diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForNestedstructureServiceDefUpdate_J10065.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNestedstructureServiceDefUpdate_J10065.java new file mode 100644 index 00000000000..d774716caf1 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForNestedstructureServiceDefUpdate_J10065.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.patch; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator.Action; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.util.CLIUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class PatchForNestedstructureServiceDefUpdate_J10065 extends BaseLoader { + private static final Logger logger = LoggerFactory.getLogger(PatchForNestedstructureServiceDefUpdate_J10065.class); + + private static final String SERVICE_DEF_NAME = EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_NESTEDSTRUCTURE_NAME; + private static final String POLICY_DOWNLOAD_AUTH_GROUPS = "policy.download.auth.groups"; + + @Autowired + ServiceDBStore svcDBStore; + + @Autowired + private RangerValidatorFactory validatorFactory; + + public static void main(String[] args) { + logger.info("main()"); + + try { + PatchForNestedstructureServiceDefUpdate_J10065 loader = (PatchForNestedstructureServiceDefUpdate_J10065) CLIUtil.getBean(PatchForNestedstructureServiceDefUpdate_J10065.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForNestedstructureServiceDefUpdate_J10065.execLoad()"); + + try { + updateNestedstructureServiceDef(); + } catch (Exception e) { + logger.error("PatchForNestedstructureServiceDefUpdate_J10065.execLoad(): failed", e); + System.exit(1); + } + + logger.info("<== PatchForNestedstructureServiceDefUpdate_J10065.execLoad()"); + } + + @Override + public void printStats() { + logger.info("PatchForNestedstructureServiceDefUpdate_J10065"); + } + + private void updateNestedstructureServiceDef() throws Exception { + logger.info("==> PatchForNestedstructureServiceDefUpdate_J10065.updateNestedstructureServiceDef()"); + + RangerServiceDef embeddedServiceDef = EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(SERVICE_DEF_NAME); + + if (embeddedServiceDef == null) { + throw new IllegalStateException("Embedded service-def does not exist: " + SERVICE_DEF_NAME); + } + + RangerServiceConfigDef embeddedConfig = findConfigByName(embeddedServiceDef.getConfigs(), POLICY_DOWNLOAD_AUTH_GROUPS); + + if (embeddedConfig == null) { + throw new IllegalStateException("Embedded service-def " + SERVICE_DEF_NAME + " does not contain config: " + POLICY_DOWNLOAD_AUTH_GROUPS); + } + + RangerServiceDef dbServiceDef = svcDBStore.getServiceDefByName(SERVICE_DEF_NAME); + + if (dbServiceDef == null) { + logger.info("Service-def [{}] does not exist in Ranger DB. Skipping.", SERVICE_DEF_NAME); + return; + } + + if (findConfigByName(dbServiceDef.getConfigs(), POLICY_DOWNLOAD_AUTH_GROUPS) != null) { + logger.info("Config [{}] already exists in service-def [{}]. Skipping.", POLICY_DOWNLOAD_AUTH_GROUPS, SERVICE_DEF_NAME); + return; + } + + List configs = CollectionUtils.isEmpty(dbServiceDef.getConfigs()) ? new ArrayList<>() : new ArrayList<>(dbServiceDef.getConfigs()); + RangerServiceConfigDef config = copyConfig(embeddedConfig); + + if (isItemIdUsed(configs, config.getItemId())) { + config.setItemId(getNextItemId(configs)); + } + + configs.add(config); + dbServiceDef.setConfigs(configs); + + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcDBStore); + + validator.validate(dbServiceDef, Action.UPDATE); + svcDBStore.updateServiceDef(dbServiceDef); + + logger.info("Added config [{}] to service-def [{}]", POLICY_DOWNLOAD_AUTH_GROUPS, SERVICE_DEF_NAME); + logger.info("<== PatchForNestedstructureServiceDefUpdate_J10065.updateNestedstructureServiceDef()"); + } + + private RangerServiceConfigDef copyConfig(RangerServiceConfigDef config) { + return new RangerServiceConfigDef(config.getItemId(), config.getName(), config.getType(), config.getSubType(), config.getMandatory(), config.getDefaultValue(), + config.getValidationRegEx(), config.getValidationMessage(), config.getUiHint(), config.getLabel(), config.getDescription(), config.getRbKeyLabel(), + config.getRbKeyDescription(), config.getRbKeyValidationMessage()); + } + + private RangerServiceConfigDef findConfigByName(List configs, String name) { + RangerServiceConfigDef ret = null; + + if (CollectionUtils.isNotEmpty(configs)) { + for (RangerServiceConfigDef config : configs) { + if (config != null && StringUtils.equals(config.getName(), name)) { + ret = config; + break; + } + } + } + + return ret; + } + + private boolean isItemIdUsed(List configs, Long itemId) { + boolean ret = false; + + if (itemId != null && CollectionUtils.isNotEmpty(configs)) { + for (RangerServiceConfigDef config : configs) { + if (config != null && itemId.equals(config.getItemId())) { + ret = true; + break; + } + } + } + + return ret; + } + + private Long getNextItemId(List configs) { + long ret = 1; + + if (CollectionUtils.isNotEmpty(configs)) { + for (RangerServiceConfigDef config : configs) { + if (config != null && config.getItemId() != null) { + ret = Math.max(ret, config.getItemId() + 1); + } + } + } + + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java index 85ab12b1581..fb094d6fb4f 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java @@ -98,6 +98,7 @@ public class RoleREST { private static List INVALID_USERS = new ArrayList<>(); public static final String POLICY_DOWNLOAD_USERS = "policy.download.auth.users"; + public static final String POLICY_DOWNLOAD_GROUPS = "policy.download.auth.groups"; public static final String PARAM_ROLE_NAME = "roleName"; public static final String PARAM_IMPORT_IN_PROGRESS = "importInProgress"; @@ -1057,12 +1058,18 @@ public RangerRoles getSecureRangerRolesIfUpdated( isAllowed = true; }else { isAllowed = bizUtil.isUserAllowed(rangerService, POLICY_DOWNLOAD_USERS); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, POLICY_DOWNLOAD_GROUPS); + } } }else{ if (isAdmin) { isAllowed = true; }else{ isAllowed = bizUtil.isUserAllowed(rangerService, POLICY_DOWNLOAD_USERS); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, POLICY_DOWNLOAD_GROUPS); + } } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index b6cffdc6878..3b880e08c03 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -162,6 +162,7 @@ public class ServiceREST { final static public String PARAM_DELETE_IF_EXISTS = "deleteIfExists"; final static public String PARAM_IMPORT_IN_PROGRESS = "importInProgress"; public static final String Allowed_User_List_For_Download = "policy.download.auth.users"; + public static final String Allowed_Group_List_For_Download = "policy.download.auth.groups"; public static final String Allowed_User_List_For_Grant_Revoke = "policy.grantrevoke.auth.users"; public static final String isCSRF_ENABLED = "ranger.rest-csrf.enabled"; @@ -3210,6 +3211,9 @@ public ServicePolicies getSecureServicePoliciesIfUpdated( } else { if (rangerService != null) { isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Download); + } if (!isAllowed) { isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); } @@ -3222,6 +3226,9 @@ public ServicePolicies getSecureServicePoliciesIfUpdated( } else { if (rangerService != null) { isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Download); + } if (!isAllowed) { isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); } @@ -4643,4 +4650,3 @@ private boolean isZoneAdmin(String zoneName) { } } - diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java index a27e59ec33c..18972280c76 100755 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java @@ -89,6 +89,7 @@ public class TagREST { private static final Logger PERF_LOG = RangerPerfTracer.getPerfLogger("rest.TagREST"); public static final String Allowed_User_List_For_Tag_Download = "tag.download.auth.users"; + public static final String Allowed_Group_List_For_Tag_Download = "tag.download.auth.groups"; @Autowired RESTErrorUtil restErrorUtil; @@ -1494,12 +1495,18 @@ public ServiceTags getSecureServiceTagsIfUpdated(@PathParam("serviceName") Strin isAllowed = true; }else { isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Tag_Download); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Tag_Download); + } } }else{ if (isAdmin) { isAllowed = true; }else{ isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Tag_Download); + if (!isAllowed) { + isAllowed = bizUtil.isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Tag_Download); + } } } if (isAllowed) { diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestRangerBizUtil.java b/security-admin/src/test/java/org/apache/ranger/biz/TestRangerBizUtil.java index 22e290a666d..9620f99be98 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestRangerBizUtil.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestRangerBizUtil.java @@ -18,7 +18,10 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.WebApplicationException; @@ -32,13 +35,16 @@ import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXAssetDao; +import org.apache.ranger.db.XXGroupDao; import org.apache.ranger.db.XXPortalUserDao; import org.apache.ranger.db.XXResourceDao; import org.apache.ranger.db.XXUserDao; import org.apache.ranger.entity.XXAsset; +import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.entity.XXResource; import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.security.context.RangerSecurityContext; import org.apache.ranger.view.VXPortalUser; @@ -652,4 +658,38 @@ public void testBlockAuditorRoleUserSuccess(){ } -} \ No newline at end of file + @Test + public void testIsUserInAllowedGroupUsesDaoGroupsForKerberosLogin() { + RangerSecurityContext context = new RangerSecurityContext(); + UserSessionBase session = new UserSessionBase(); + XXPortalUser portalUser = new XXPortalUser(); + + portalUser.setLoginId("hive/lapa-adh2-latest-1.ru-central1.internal@RU-CENTRAL1.INTERNAL"); + session.setXXPortalUser(portalUser); + context.setUserSession(session); + RangerContextHolder.setSecurityContext(context); + + RangerService service = new RangerService(); + Map configs = new HashMap<>(); + configs.put("policy.download.auth.groups", "spark_users"); + service.setConfigs(configs); + + XXUserDao userDao = Mockito.mock(XXUserDao.class); + XXGroupDao groupDao = Mockito.mock(XXGroupDao.class); + XXUser xUser = new XXUser(); + xUser.setId(7L); + xUser.setName("hive"); + XXGroup xGroup = new XXGroup(); + xGroup.setName("spark_users"); + + Mockito.when(daoManager.getXXUser()).thenReturn(userDao); + Mockito.when(daoManager.getXXGroup()).thenReturn(groupDao); + Mockito.when(userDao.findByUserName("hive")).thenReturn(xUser); + Mockito.when(groupDao.findByUserId(7L)).thenReturn(Collections.singletonList(xGroup)); + + Assert.assertTrue(rangerBizUtil.isUserInAllowedGroup(service, "policy.download.auth.groups")); + Mockito.verify(userDao).findByUserName("hive"); + Mockito.verify(groupDao).findByUserId(7L); + } + +} diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java index 72dc33b291c..9c6ccb2482a 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java @@ -26,6 +26,7 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerRole; +import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.validation.RangerRoleValidator; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; import org.apache.ranger.plugin.util.RangerRoles; @@ -816,6 +817,49 @@ public void test17dGetSecureRangerRolesIfUpdated(){ } } + @Test + public void test17eGetSecureRangerRolesIfUpdatedAllowedByDownloadGroup() throws Exception { + RangerRoles rangerRoles = createRangerRoles(); + rangerRoles.setRoleVersion(2L); + + String serviceName = "serviceName"; + String pluginId = "pluginId"; + String clusterName = ""; + String pluginCapabilities = ""; + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + XXService xxService = new XXService(); + xxService.setId(1L); + xxService.setName(serviceName); + xxService.setType(1L); + + XXServiceDef xxServiceDef = new XXServiceDef(); + xxServiceDef.setId(1L); + xxServiceDef.setImplclassname("org.apache.ranger.services.hdfs.RangerServiceHdfs"); + + RangerService rangerService = new RangerService(); + rangerService.setName(serviceName); + + Mockito.when(serviceUtil.isValidService(serviceName, request)).thenReturn(true); + Mockito.when(daoMgr.getXXService().findByName(serviceName)).thenReturn(xxService); + Mockito.when(daoMgr.getXXServiceDef().getById(xxService.getType())).thenReturn(xxServiceDef); + Mockito.when(svcStore.getServiceByName(serviceName)).thenReturn(rangerService); + Mockito.when(bizUtil.isAdmin()).thenReturn(false); + Mockito.when(bizUtil.isKeyAdmin()).thenReturn(false); + Mockito.when(bizUtil.isUserAllowed(rangerService, RoleREST.POLICY_DOWNLOAD_USERS)).thenReturn(false); + Mockito.when(bizUtil.isUserInAllowedGroup(rangerService, RoleREST.POLICY_DOWNLOAD_GROUPS)).thenReturn(true); + Mockito.when(roleStore.getRoles(serviceName, -1L)).thenReturn(rangerRoles); + + RangerRoles returnedRoles = roleRest.getSecureRangerRolesIfUpdated(serviceName, -1L, 0L, pluginId, + clusterName, pluginCapabilities, request); + + Assert.assertNotNull(returnedRoles); + Assert.assertEquals(serviceName, returnedRoles.getServiceName()); + Mockito.verify(bizUtil).isUserAllowed(rangerService, RoleREST.POLICY_DOWNLOAD_USERS); + Mockito.verify(bizUtil).isUserInAllowedGroup(rangerService, RoleREST.POLICY_DOWNLOAD_GROUPS); + Mockito.verify(roleStore).getRoles(serviceName, -1L); + } + // empty request roles (requestParamRoles = 0, dbRoles = 5, return = all dbRoles) @Test public void test18GetRolesInJson() throws Exception { diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java index 290d29933c9..381e3a1860f 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java @@ -2112,6 +2112,41 @@ public void test59getSecureServicePoliciesIfUpdatedSuccess() throws Exception { Mockito.verify(svcStore).getServicePoliciesIfUpdated(serviceName, lastKnownVersion, false); } + @Test + public void test59aGetSecureServicePoliciesIfUpdatedAllowedByDownloadGroup() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + Long lastKnownVersion = 1L; + String pluginId = "1"; + XXService xService = xService(); + XXServiceDef xServiceDef = serviceDef(); + String serviceName = xService.getName(); + RangerService rs = rangerService(); + ServicePolicies sp = servicePolicies(); + XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class); + + Mockito.when(bizUtil.isAdmin()).thenReturn(false); + Mockito.when(bizUtil.isKeyAdmin()).thenReturn(false); + Mockito.when(serviceUtil.isValidService(serviceName, request)).thenReturn(true); + Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); + Mockito.when(xServiceDao.findByName(serviceName)).thenReturn(xService); + Mockito.when(daoManager.getXXServiceDef()).thenReturn(xServiceDefDao); + Mockito.when(xServiceDefDao.getById(xService.getType())).thenReturn(xServiceDef); + Mockito.when(svcStore.getServiceByName(serviceName)).thenReturn(rs); + Mockito.when(bizUtil.isUserAllowed(rs, ServiceREST.Allowed_User_List_For_Download)).thenReturn(false); + Mockito.when(bizUtil.isUserInAllowedGroup(rs, ServiceREST.Allowed_Group_List_For_Download)).thenReturn(true); + Mockito.when(svcStore.getServicePoliciesIfUpdated(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(sp); + + ServicePolicies dbServiceSecurePolicies = serviceREST.getSecureServicePoliciesIfUpdated(serviceName, + lastKnownVersion, 0L, pluginId, "", "", true, capabilityVector, request); + + Assert.assertNotNull(dbServiceSecurePolicies); + Mockito.verify(bizUtil).isUserAllowed(rs, ServiceREST.Allowed_User_List_For_Download); + Mockito.verify(bizUtil).isUserInAllowedGroup(rs, ServiceREST.Allowed_Group_List_For_Download); + Mockito.verify(bizUtil, Mockito.never()).isUserAllowed(rs, ServiceREST.Allowed_User_List_For_Grant_Revoke); + Mockito.verify(svcStore).getServicePoliciesIfUpdated(serviceName, lastKnownVersion, false); + } + @Test public void test60getPolicyFromEventTime() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestTagREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestTagREST.java index 34122dd7998..aee36c101b3 100755 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestTagREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestTagREST.java @@ -76,6 +76,7 @@ public class TestTagREST { private static Long lastKnownVersion = 10L; private static String pluginId = "1"; private static String Allowed_User_List_For_Tag_Download = "tag.download.auth.users"; + private static String Allowed_Group_List_For_Tag_Download = "tag.download.auth.groups"; @InjectMocks TagREST tagREST = new TagREST(); @@ -1848,6 +1849,55 @@ public void test55getSecureServiceTagsIfUpdatedIsAdminFalse() { } catch (Exception e) { } } + + @Test + public void test55aGetSecureServiceTagsIfUpdatedAllowedByDownloadGroup() { + boolean isAdmin = false; + boolean isKeyAdmin = false; + ServiceTags oldServiceTag = new ServiceTags(); + oldServiceTag.setServiceName(serviceName); + oldServiceTag.setTagVersion(5L); + + XXService xService = new XXService(); + xService.setId(id); + xService.setName(serviceName); + xService.setType(5L); + + XXServiceDef xServiceDef = new XXServiceDef(); + xServiceDef.setId(id); + xServiceDef.setVersion(5L); + + RangerService rangerService = new RangerService(); + rangerService.setId(id); + rangerService.setName(serviceName); + + XXServiceDao xXServiceDao = Mockito.mock(XXServiceDao.class); + XXServiceDefDao xXServiceDefDao = Mockito.mock(XXServiceDefDao.class); + + Mockito.when(bizUtil.isAdmin()).thenReturn(isAdmin); + Mockito.when(bizUtil.isKeyAdmin()).thenReturn(isKeyAdmin); + Mockito.when(daoManager.getXXService()).thenReturn(xXServiceDao); + Mockito.when(xXServiceDao.findByName(serviceName)).thenReturn(xService); + Mockito.when(daoManager.getXXServiceDef()).thenReturn(xXServiceDefDao); + Mockito.when(xXServiceDefDao.getById(xService.getType())).thenReturn(xServiceDef); + + try { + Mockito.when(svcStore.getServiceByName(serviceName)).thenReturn(rangerService); + Mockito.when(tagStore.getServiceTagsIfUpdated(serviceName, lastKnownVersion, true)).thenReturn(oldServiceTag); + } catch (Exception e) { + } + + Mockito.when(bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Tag_Download)).thenReturn(false); + Mockito.when(bizUtil.isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Tag_Download)).thenReturn(true); + + ServiceTags result = tagREST.getSecureServiceTagsIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, false, capabilityVector, null); + + Assert.assertNotNull(result.getServiceName()); + Assert.assertEquals(result.getServiceName(), oldServiceTag.getServiceName()); + Assert.assertEquals(result.getTagVersion(), oldServiceTag.getTagVersion()); + Mockito.verify(bizUtil).isUserAllowed(rangerService, Allowed_User_List_For_Tag_Download); + Mockito.verify(bizUtil).isUserInAllowedGroup(rangerService, Allowed_Group_List_For_Tag_Download); + } @Test public void test56getSecureServiceTagsIfUpdatedIsAllowedFalse() {