diff --git a/authentication/pom.xml b/authentication/pom.xml new file mode 100644 index 000000000..a1243f56a --- /dev/null +++ b/authentication/pom.xml @@ -0,0 +1,104 @@ + + + + 4.0.0 + + com.yahoo.oozie + oozie-main + 2.3.0-SNAPSHOT + + oozie-authentication + Oozie HTTP authentication + Oozie HTTP authentication + jar + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + Yahoo + http://www.yahoo.com + + + + + commons-codec + commons-codec + 1.4 + compile + + + + xmlenc + xmlenc + 0.52 + compile + + + + org.slf4j + slf4j-api + 1.4.3 + + + + + com.yahoo.hadoop + hadoop-core + provided + + + + org.testng + testng + 5.8 + jdk15 + test + + + + org.mockito + mockito-core + 1.7 + test + + + + org.easymock + easymock + 2.4 + test + + + + + + + maven-assembly-plugin + + + ../src/main/assemblies/empty.xml + + + + + + + diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/AbstractAuthenticationToken.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/AbstractAuthenticationToken.java new file mode 100644 index 000000000..c4c24f39d --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/AbstractAuthenticationToken.java @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +/** + * Abstract class contains basic implementation for {@link AuthenticationToken}. + *

+ * To provide user-defined authentication, one token AuthenticationToken + * instance has to be provided to carry the information that authentication needs. + */ +public abstract class AbstractAuthenticationToken implements AuthenticationToken { + + private boolean authenticated = false; + protected String principal; + protected String remoteAddr; + private final String token; + + protected AbstractAuthenticationToken(String remoteAddr, String token) { + this(null, remoteAddr, token); + } + + protected AbstractAuthenticationToken(String principal, String remoteAddr, String token) { + this.principal = principal; + this.remoteAddr = remoteAddr; + this.token = token; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.AuthenticationToken#isAuthenticated() + */ + @Override + public boolean isAuthenticated() { + return authenticated; + } + + /** + * Set true if token is authenticated + * + * @param authenticated + */ + public void setAuthenticated(boolean authenticated) { + this.authenticated = authenticated; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.AuthenticationToken#getPrincipal() + */ + @Override + public String getPrincipal() { + return principal; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.AuthenticationToken#getRemoteAddr() + */ + @Override + public String getRemoteAddr() { + return remoteAddr; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.AuthenticationToken#getToken() + */ + public String getToken() { + return token; + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProvider.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProvider.java new file mode 100644 index 000000000..21e43e10c --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProvider.java @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +/** + * Provider interface for server authentication class. + * + */ +public interface AuthenticationProvider { + + /** + * Check if this authentication provider supports client request + * + * @param httpServletRequest + * @return true if this authentication provider supports client request + */ + boolean supports(HttpServletRequest httpServletRequest); + + /** + * Get authentication token + * + * @param httpServletRequest httpServletRequest + * @return authentication token + * @throws IOException thrown if error to retrieve token from request + */ + AuthenticationToken getAuthenticationToken(HttpServletRequest httpServletRequest) throws IOException; + + /** + * Verify and authenticate the token + * + * @param authenticationToken authentication token + * @return authentication token + */ + AuthenticationToken authenticate(AuthenticationToken authenticationToken); +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProviderFactory.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProviderFactory.java new file mode 100644 index 000000000..84d53933c --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationProviderFactory.java @@ -0,0 +1,114 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.exception.AuthenticationException; +import org.apache.hadoop.http.authentication.exception.UnknownAuthenticationSchemeException; +import org.apache.hadoop.http.authentication.web.util.Assert; + +import javax.servlet.http.HttpServletRequest; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; + +/** + * This class is instantiated by {@link AuthenticationProcessingFilter} to statically store all providers in a map. + *

+ * The parameter "authentication.providers" is used to specify a list of providers server can support. Filter + * {@link AuthenticationProcessingFilter} use this factory to get appropriate provider to authenticate request. + * + */ +public class AuthenticationProviderFactory { + + private List authenticationProviderList = new ArrayList(); + + public AuthenticationProviderFactory(Configuration configuration) { + initializeAuthenticationProviders(configuration); + } + + /** + * Initialize a list of providers specified in configuration "authentication.providers". + *

+ * A list of providers is stored in a map for further use. + * + * @param configuration configuration contains parameters to instantiate providers + */ + private void initializeAuthenticationProviders(final Configuration configuration) { + String authProviderConf = configuration.get("authentication.providers"); + Assert.notNull(authProviderConf, + "You should configure at least one authentication provider in authentication.providers"); + String[] authenticationProviders = authProviderConf.split("\\s*,\\s*"); + + for (String authenticationProviderFQNClassName : authenticationProviders) { + try { + Class providerClass = Class.forName(authenticationProviderFQNClassName); + Constructor constructor = providerClass.getDeclaredConstructor(new Class[] { Configuration.class }); + + AuthenticationProvider provider = (AuthenticationProvider) constructor.newInstance(configuration); + authenticationProviderList.add(provider); + } + catch (InstantiationException e) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, e); + } + catch (IllegalAccessException e) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, e); + } + catch (ClassNotFoundException e) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, e); + } + catch (NoSuchMethodException e) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, e); + } + catch (InvocationTargetException e) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, e); + } + catch (Throwable t) { + throw new AuthenticationException("Unable to create instance for: " + + authenticationProviderFQNClassName, t); + } + } + } + + /** + * Get authentication provider if supports() return true + * + * @param httpServletRequest httpServletRequest + * @return authentication provider if supports() return true + */ + public AuthenticationProvider getAuthenticationProvider(HttpServletRequest httpServletRequest) { + AuthenticationProvider supportedProvider = null; + + for (AuthenticationProvider authenticationProvider : authenticationProviderList) { + if (authenticationProvider.supports(httpServletRequest)) { + supportedProvider = authenticationProvider; + break; + } + } + + if (supportedProvider == null) { + throw new UnknownAuthenticationSchemeException("None of the configured providers could " + + "identify a scheme in Request."); + } + + return supportedProvider; + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationToken.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationToken.java new file mode 100644 index 000000000..7d4a98c88 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationToken.java @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +/** + * The interface for a token which contains the information for a authentication provider to use. + *

+ * Abstract class {@link} provides basic implementation for this interface. + * + */ +public interface AuthenticationToken { + + /** + * Get authentication name + * + * @return authentication name + */ + String getAuthenticationMethod(); + + /** + * True if it is authenticated + * + * @return true if authenticated + */ + boolean isAuthenticated(); + + /** + * Get Principal + * + * @return principal + */ + String getPrincipal(); + + /** + * Get remote address + * + * @return remote address + */ + String getRemoteAddr(); + + /** + * Get token + * + * @return token + */ + String getToken(); +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationTokenSerDe.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationTokenSerDe.java new file mode 100644 index 000000000..9ec681db6 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/AuthenticationTokenSerDe.java @@ -0,0 +1,152 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.http.authentication.exception.AccessDeniedException; +import org.apache.hadoop.http.authentication.web.CookieSignerVerifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.GeneralSecurityException; +import java.util.concurrent.TimeUnit; + +/** + * The AuthenticationTokenSerDe is used to serialize and deserialize authentication token. + * + */ +public class AuthenticationTokenSerDe { + + private static final Logger LOGGER = LoggerFactory.getLogger(AuthenticationTokenSerDe.class); + private static final String DELIMITER = ";"; + private static final String SIGN_KEY_STRING = "sign="; + public static final long EXPIRATION_IN_MILLIS = TimeUnit.HOURS.toMillis(4); + public static final long EXPIRATION_IN_SECONDS = TimeUnit.HOURS.toSeconds(4); + + private AuthenticationTokenSerDe() { + } + + /** + * Serialize a token and encode token by signature CookieSignerVerifier + * + * @param authenticationToken authentication token + * @param cookieSigner signature verifier + * @return serialized string + */ + public static String serialize(AuthenticationToken authenticationToken, CookieSignerVerifier cookieSigner) { + StringBuilder buffer = new StringBuilder(); + buffer.append(authenticationToken.getAuthenticationMethod()).append(DELIMITER); + buffer.append(authenticationToken.getPrincipal()).append(DELIMITER); + buffer.append(authenticationToken.getRemoteAddr()).append(DELIMITER); + buffer.append(System.currentTimeMillis() + EXPIRATION_IN_MILLIS); + + try { + final String token = buffer.toString(); + String signature = cookieSigner.getSignature(token); + String serializedToken = Base64.encodeBase64URLSafeString(token.getBytes()) + SIGN_KEY_STRING + signature; + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Created cookie: " + serializedToken); + } + return serializedToken; + } + catch (GeneralSecurityException e) { + throw new IllegalStateException("Error while trying to sign the cookie", e); + } + } + + /** + * Deserialize a token and verify signature by CookieSignerVerifier + * + * @param serializedToken serialized token + * @param remoteAddr remote address + * @param cookieVerifier signature verifier + * @return authentication token + */ + public static AuthenticationToken deserialize(String serializedToken, String remoteAddr, + CookieSignerVerifier cookieVerifier) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Received cookie for deserialize: " + serializedToken); + } + int index = serializedToken.lastIndexOf(SIGN_KEY_STRING); + if (index == -1) + throw new AccessDeniedException("Authentication cookie received does not contain signature"); + String signature = serializedToken.substring(index + SIGN_KEY_STRING.length()); + String data = serializedToken.substring(0, index); + String decodedData = new String(Base64.decodeBase64(data)); + + try { + if (!cookieVerifier.verifySignature(decodedData, signature)) + throw new AccessDeniedException("Authentication cookie received failed signature verification"); + } + catch (GeneralSecurityException e) { + throw new AccessDeniedException("Error while trying to verify the authentication cookie received", e); + } + + String[] tokens = decodedData.split(DELIMITER); + if (tokens.length != 4) { + throw new AccessDeniedException("Authentication cookie received is invalid"); + } + + CookieBasedAuthenticationToken cookieToken = new CookieBasedAuthenticationToken(tokens[0], tokens[1], tokens[2]); + checkTokenForExpiry(tokens[3], cookieToken); + checkTokenForRemoteAddr(cookieToken, remoteAddr); + return cookieToken; + } + + private static void checkTokenForExpiry(String expirationTimeInMillisStr, CookieBasedAuthenticationToken cookieToken) { + final long expirationTimeInMillis = Long.valueOf(expirationTimeInMillisStr); + final boolean isExpired = expirationTimeInMillis < System.currentTimeMillis(); + + if (isExpired) { + String msg = "Received expired authentication token"; + LOGGER.error(msg + ". Cookie info: " + cookieToken.toString()); + throw new AccessDeniedException(msg); + } + } + + private static void checkTokenForRemoteAddr(CookieBasedAuthenticationToken cookieToken, String remoteAddr) { + if (!remoteAddr.equals(cookieToken.getRemoteAddr())) { + String msg = "Authentication cookie received was issued to " + cookieToken.getRemoteAddr() + + ". But the cookie was received from " + remoteAddr; + LOGGER.error(msg + ". Cookie info: " + cookieToken.toString()); + throw new AccessDeniedException(msg); + } + } + + public static class CookieBasedAuthenticationToken extends AbstractAuthenticationToken { + + private final String initialAuthMethod; + + CookieBasedAuthenticationToken(String initialAuthMethod, String principal, String remoteAddr) { + super(principal, remoteAddr, null); + super.setAuthenticated(true); + this.initialAuthMethod = initialAuthMethod; + } + + @Override + public String getAuthenticationMethod() { + return "Cookie"; + } + + public String getInitialAuthMethod() { + return initialAuthMethod; + } + + @Override + public String toString() { + return "User = " + principal + ",RemoteIP = " + remoteAddr + ",InitalAuthMethod = " + initialAuthMethod; + } + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/TicketValidator.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/TicketValidator.java new file mode 100644 index 000000000..e6abcfb0d --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/TicketValidator.java @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication; + +import org.apache.hadoop.http.authentication.exception.BadCredentialsException; + +/** + * Interface to implement to validate authentication token AuthenticationToken + * + */ +public interface TicketValidator { + /** + * Validate token + * + * @param token + * @return string + * @throws BadCredentialsException + */ + public String validateTicket(AuthenticationToken token) throws BadCredentialsException; +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/client/HttpAuthenticator.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/HttpAuthenticator.java new file mode 100644 index 000000000..6db8305f1 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/HttpAuthenticator.java @@ -0,0 +1,145 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.client; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.servlet.http.Cookie; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An authenticator abstract class to provide interfaces and common implementation for client side authentication. + */ +public abstract class HttpAuthenticator { + + private static final String HADOOP_HTTP_AUTH = "Hadoop-HTTP-Auth"; + public static final String CONF_FOR_HTTP_RENEW_AUTHENTICATION = "http.renew.authentication"; + private static final Logger LOGGER = LoggerFactory.getLogger(HttpAuthenticator.class.getName()); + protected Cookie authCookie = null; + private long cookieExpiryTime = 0L; + + private static final ThreadLocal cookieExpiresFormat = new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("E, dd-MMM-yyyy k:m:s z"); + } + }; + + /** + * To authenticate a client request for http connection + * + * @param conf the configuration for authentication + * @param connection the connection to authenticate + * @throws IOException + */ + public abstract void authenticate(Map conf, HttpURLConnection connection) throws IOException; + + /** + * Read cookie from http response + * + * @param connection http connection + */ + public void setCookieFromResponse(HttpURLConnection connection) { + Map> headers = connection.getHeaderFields(); + List cookieHeaders = headers.get("Set-Cookie"); + try { + if (cookieHeaders != null) { + for (String cookieString : cookieHeaders) { + String[] attributes = cookieString.split(";"); + String nameValue = attributes[0]; + int equals = nameValue.indexOf('='); + String cookieName = nameValue.substring(0, equals); + if (cookieName.equals(HADOOP_HTTP_AUTH)) { + String cookieValue = nameValue.substring(equals + 1); + + for (int i = 1; i < attributes.length; i++) { + nameValue = attributes[i].trim(); + if ((equals = nameValue.indexOf('=')) == -1) + continue; + String attributeName = nameValue.substring(0, equals); + String attributeValue = nameValue.substring(equals + 1); + if (attributeName.equalsIgnoreCase("expires")) { + long expiryTime = cookieExpiresFormat.get().parse(attributeValue).getTime(); + if (expiryTime < cookieExpiryTime) { + clearCookie(); + } + else { + authCookie = new Cookie(cookieName, cookieValue); + cookieExpiryTime = expiryTime; + } + break; + } + } + } + } + } + } + catch (Exception e) { + LOGGER.warn("Failed to read cookie from response", e); + } + } + + /** + * Clear the cookie + */ + public void clearCookie() { + authCookie = null; + cookieExpiryTime = 0L; + } + + /** + * Set cookie to http request + * + * @param connection http connection + * @return true if cookie is added to http request + */ + protected boolean setCookieInRequest(HttpURLConnection connection) { + if (authCookie == null || hasCookieExpired()) + return false; + connection.addRequestProperty("Cookie", authCookie.getName() + "=" + authCookie.getValue()); + return true; + } + + /** + * Check if cookie is expired + * + * @return true if the Cookie has expired or 10 mins to expire + */ + private boolean hasCookieExpired() { + if ((cookieExpiryTime - System.currentTimeMillis()) <= TimeUnit.MINUTES.toMillis(10)) { + authCookie = null; + cookieExpiryTime = 0L; + return true; + } + return false; + } + + /** + * Return boolean value + * + * @param value string boolean value + * @return true if String "true" is given + */ + protected boolean getBooleanValue(String value) { + return value != null && value.equals("true"); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/client/NullAuthenticator.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/NullAuthenticator.java new file mode 100644 index 000000000..3608926b7 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/NullAuthenticator.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.client; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Map; + +/** + * NO-OP authenticator + * + */ +public class NullAuthenticator extends HttpAuthenticator { + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.client.HttpAuthenticator#authenticate(java.util.Map, java.net.HttpURLConnection) + */ + @Override + public void authenticate(Map conf, HttpURLConnection connection) throws IOException { + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/client/simple/SimpleAuthenticator.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/simple/SimpleAuthenticator.java new file mode 100644 index 000000000..94bb5a351 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/client/simple/SimpleAuthenticator.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.client.simple; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Map; + +import org.apache.hadoop.http.authentication.client.HttpAuthenticator; + +/** + * Authenticator class for simple authentication. + */ +public class SimpleAuthenticator extends HttpAuthenticator { + public static final String REQUEST_PARAMETER_NAME = "ugi"; + + /** + * Get property "ugi" from http connection. This is simple authentication for default use. + * + * @see org.apache.hadoop.http.authentication.client.HttpAuthenticator#authenticate(java.util.Map, java.net.HttpURLConnection) + */ + @Override + public void authenticate(Map conf, HttpURLConnection connection) throws IOException { + connection.setRequestProperty(REQUEST_PARAMETER_NAME, System.getProperty("user.name")); + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AccessDeniedException.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AccessDeniedException.java new file mode 100644 index 000000000..9a56f2b86 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AccessDeniedException.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.exception; + +public class AccessDeniedException extends AuthenticationException { + + private static final long serialVersionUID = 1L; + + public AccessDeniedException(String message) { + super(message); + } + + public AccessDeniedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AuthenticationException.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AuthenticationException.java new file mode 100644 index 000000000..5cdd0c842 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/AuthenticationException.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.exception; + +public class AuthenticationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public AuthenticationException(String message) { + super(message); + } + + public AuthenticationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/BadCredentialsException.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/BadCredentialsException.java new file mode 100644 index 000000000..47be77d90 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/BadCredentialsException.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.exception; + +public class BadCredentialsException extends AuthenticationException { + + private static final long serialVersionUID = 1L; + + public BadCredentialsException(String message) { + super(message); + } + + public BadCredentialsException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/UnknownAuthenticationSchemeException.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/UnknownAuthenticationSchemeException.java new file mode 100644 index 000000000..575666966 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/exception/UnknownAuthenticationSchemeException.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.exception; + +public class UnknownAuthenticationSchemeException extends AuthenticationException { + + private static final long serialVersionUID = 1L; + + public UnknownAuthenticationSchemeException(String message) { + super(message); + } + + public UnknownAuthenticationSchemeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationHeaderProvider.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationHeaderProvider.java new file mode 100644 index 000000000..6aad3b34e --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationHeaderProvider.java @@ -0,0 +1,85 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.server.simple; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.AuthenticationProvider; +import org.apache.hadoop.http.authentication.AuthenticationToken; +import org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator; +import org.apache.hadoop.http.authentication.exception.AccessDeniedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provider class for simple authentication. + */ +public class SimpleAuthenticationHeaderProvider implements AuthenticationProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleAuthenticationHeaderProvider.class); + + public SimpleAuthenticationHeaderProvider(Configuration configuration) { + } + + /** + * Check if parameter "ugi" exists in http request header. True if it exists. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#supports(javax.servlet.http.HttpServletRequest) + */ + @Override + public boolean supports(HttpServletRequest httpServletRequest) { + final String header = httpServletRequest.getHeader(SimpleAuthenticator.REQUEST_PARAMETER_NAME); + + if (header != null) { + if (LOGGER.isDebugEnabled()) { + LOGGER + .debug("Received UGI parameter for request " + httpServletRequest.getRequestURL() + ": " + + header); + } + + return true; + } + + return false; + } + + /** + * Get parameter "ugi" and initialize SimpleAuthenticationToken. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#getAuthenticationToken(javax.servlet.http.HttpServletRequest) + */ + @Override + public AuthenticationToken getAuthenticationToken(HttpServletRequest httpServletRequest) throws IOException { + final String header = httpServletRequest.getHeader(SimpleAuthenticator.REQUEST_PARAMETER_NAME); + + if (header != null) { + return new SimpleAuthenticationToken(header, httpServletRequest.getRemoteAddr(), null); + } + + throw new AccessDeniedException("Received UGI in request with no value."); + } + + /** + * Authenticate token for simple authentication. It is no-op in SimpleAuthenticationHeaderProvider. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#authenticate(org.apache.hadoop.http.authentication.AuthenticationToken) + */ + @Override + public AuthenticationToken authenticate(AuthenticationToken authenticationToken) { + return authenticationToken; + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationProvider.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationProvider.java new file mode 100644 index 000000000..4da6fd36b --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.server.simple; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.AuthenticationProvider; +import org.apache.hadoop.http.authentication.AuthenticationToken; +import org.apache.hadoop.http.authentication.exception.AccessDeniedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +/** + * A simple authentication to verify if http request has "ugi" parameter which contains requester's user name. + */ +public class SimpleAuthenticationProvider implements AuthenticationProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleAuthenticationProvider.class); + + private static final String REQUEST_PARAMETER_NAME = "ugi"; + + public SimpleAuthenticationProvider(Configuration configuration) { + } + + /** + * True if the http request contains parameter 'ugi'. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#supports(javax.servlet.http.HttpServletRequest) + */ + @Override + public boolean supports(HttpServletRequest httpServletRequest) { + final String parameter = httpServletRequest.getParameter(REQUEST_PARAMETER_NAME); + + if (parameter != null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Received UGI parameter for request " + httpServletRequest.getRequestURL() + ": " + + parameter); + } + + return true; + } + + return false; + } + + /** + * Get 'ugi' from http request and construct a token object. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#getAuthenticationToken(javax.servlet.http.HttpServletRequest) + */ + @Override + public AuthenticationToken getAuthenticationToken(HttpServletRequest httpServletRequest) throws IOException { + final String parameter = httpServletRequest.getParameter(REQUEST_PARAMETER_NAME); + + if (parameter != null) { + return new SimpleAuthenticationToken(parameter, httpServletRequest.getRemoteAddr(), null); + } + + throw new AccessDeniedException("Received UGI in request with no value."); + } + + /** + * This method is to authenticate the token. It is no-op in SimpleAuthenticationProvider. + * + * @see org.apache.hadoop.http.authentication.AuthenticationProvider#authenticate(org.apache.hadoop.http.authentication.AuthenticationToken) + */ + @Override + public AuthenticationToken authenticate(AuthenticationToken authenticationToken) { + return authenticationToken; + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationToken.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationToken.java new file mode 100644 index 000000000..291b0fe75 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/server/simple/SimpleAuthenticationToken.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.server.simple; + +import org.apache.hadoop.http.authentication.AbstractAuthenticationToken; + +/** + * A token contains basic functionality to support simple authentication. + * + */ +public class SimpleAuthenticationToken extends AbstractAuthenticationToken { + + private static final String AUTHENTICATION_METHOD = "simple"; + + public SimpleAuthenticationToken(String remoteAddr, String token) { + super(remoteAddr, token); + } + + public SimpleAuthenticationToken(String principal, String remoteAddr, String token) { + super(principal, remoteAddr, token); + super.setAuthenticated(true); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.http.authentication.AuthenticationToken#getAuthenticationMethod() + */ + @Override + public String getAuthenticationMethod() { + return AUTHENTICATION_METHOD; + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationHelper.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationHelper.java new file mode 100644 index 000000000..dc4f316ca --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationHelper.java @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import org.apache.hadoop.http.authentication.AuthenticationToken; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +import java.io.IOException; +import java.security.Principal; + +/** + * Provide utility functions for authentication. + * + */ +final class AuthenticationHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(AuthenticationHelper.class.getName()); + + private AuthenticationHelper() { + } + + /** + * Get ugi from cache ProxyUGICacheManager or create it if not existed. + * + * @param ugiManager the container to store ugi + * @param httpServletRequest http request + * @param authenticatedToken authenticated token + * @throws IOException thrown if failed to get ugi + */ + static void setupUGI(ProxyUGICacheManager ugiManager, HttpServletRequest httpServletRequest, + AuthenticationToken authenticatedToken) throws IOException { + UserGroupInformation proxyUGI = ugiManager.getUGI(authenticatedToken.getPrincipal(), httpServletRequest); + httpServletRequest.setAttribute("authorized.ugi", proxyUGI); + LOGGER.info("Proxying as " + authenticatedToken.getPrincipal()); + } + + /** + * Create a http request HttpServletRequestWrapper by overriding getUserPrincipal() and getRemoteUser() + * to return user name from authentication token. + * + * @param httpServletRequest http request + * @param authenticatedToken authenticated token + * @return http request wrapper + */ + static HttpServletRequestWrapper createAuthenticatedRequest(final HttpServletRequest httpServletRequest, + final AuthenticationToken authenticatedToken) { + + return new HttpServletRequestWrapper(httpServletRequest) { + @Override + public String getRemoteUser() { + return authenticatedToken.getPrincipal(); + } + + @Override + public Principal getUserPrincipal() { + return new Principal() { + + @Override + public String getName() { + return authenticatedToken.getPrincipal(); + } + }; + } + }; + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationProcessingFilter.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationProcessingFilter.java new file mode 100644 index 000000000..4da3c1520 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/AuthenticationProcessingFilter.java @@ -0,0 +1,188 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.AuthenticationProvider; +import org.apache.hadoop.http.authentication.AuthenticationProviderFactory; +import org.apache.hadoop.http.authentication.AuthenticationToken; +import org.apache.hadoop.http.authentication.AuthenticationTokenSerDe; +import org.apache.hadoop.http.authentication.AuthenticationTokenSerDe.CookieBasedAuthenticationToken; +import org.apache.hadoop.http.authentication.exception.AccessDeniedException; +import org.apache.hadoop.http.authentication.exception.AuthenticationException; +import org.apache.hadoop.http.authentication.exception.UnknownAuthenticationSchemeException; +import org.apache.hadoop.http.exception.HttpExceptionUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +/** + * The filter AuthenticationProcessingFilter delegate to different authentication mechanisms to authenticate + * http requests. When web resources are gated by AuthenticationProcessingFilter, requests first come to + * here to do authentication and continue on the rest of filters before reaching the resources. + * + * These resources needed in AuthenticationProcessingFilter are saved in context as : + *

+ */ +public class AuthenticationProcessingFilter implements javax.servlet.Filter { + + public static final String AUTH_CONFIGURATION = "auth.configuration"; + public static final String UGI_CACHE_MANAGER = "ugi.cache.manager"; + public static final String COOKIE_SIGNER_VERIFIER = "cookie.signer.verifier"; + public static final String AUTHENTICATION_TOKEN = "authentication.token"; + private static final Logger LOGGER = LoggerFactory.getLogger(AuthenticationProcessingFilter.class.getName()); + + private AuthenticationProviderFactory providerFactory; + private ProxyUGICacheManager ugiManager; + private CookieSignerVerifier cookieSignerVerifier; + + /* (non-Javadoc) + * @see javax.servlet.Filter#init(javax.servlet.FilterConfig) + */ + public void init(FilterConfig config) throws ServletException { + Configuration configuration = (Configuration) config.getServletContext().getAttribute(AUTH_CONFIGURATION); + providerFactory = new AuthenticationProviderFactory(configuration); + ugiManager = (ProxyUGICacheManager) config.getServletContext().getAttribute(UGI_CACHE_MANAGER); + cookieSignerVerifier = (CookieSignerVerifier) config.getServletContext().getAttribute(COOKIE_SIGNER_VERIFIER); + } + + /* (non-Javadoc) + * @see javax.servlet.Filter#doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) + */ + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws ServletException, IOException { + final HttpServletRequest httpServletRequest = (HttpServletRequest) servletRequest; + final HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; + final String path = httpServletRequest.getPathInfo(); + AuthenticationToken authenticatedToken = null; + AuthenticationException cookieException = null; + try { + try { + authenticatedToken = getAuthenticationTokenFromRequest(httpServletRequest); + } + catch (AuthenticationException e) { + cookieException = e; + } + if (authenticatedToken == null || !authenticatedToken.isAuthenticated()) { + authenticatedToken = authenticate(httpServletRequest); + MDC.put("User", authenticatedToken.getPrincipal()); + Cookie cookie = CookieHelper.create(AuthenticationTokenSerDe.serialize(authenticatedToken, + cookieSignerVerifier)); + httpServletResponse.addCookie(cookie); + } + httpServletRequest.setAttribute(AUTHENTICATION_TOKEN, authenticatedToken); + AuthenticationHelper.setupUGI(ugiManager, httpServletRequest, authenticatedToken); + ServletRequest authenticatedRequest = AuthenticationHelper.createAuthenticatedRequest(httpServletRequest, + authenticatedToken); + + filterChain.doFilter(authenticatedRequest, servletResponse); + + } + catch (UnknownAuthenticationSchemeException ignore) { + httpServletResponse.addHeader("WWW-Authenticate", "Negotiate"); + if (cookieException == null) { + LOGGER.warn("Request did not have any authentication information. Replying with Negotiate header", + ignore); + HttpExceptionUtil.sendErrorAsXml(httpServletResponse, HttpServletResponse.SC_UNAUTHORIZED, + new AccessDeniedException("Authentication is required"), path); + } + else { + handleAuthenticationFailure(httpServletResponse, cookieException, path); + } + } + catch (AuthenticationException e) { + handleAuthenticationFailure(httpServletResponse, e, path); + } + catch (IllegalArgumentException e) { + sendErrorAsXml(httpServletResponse, e, HttpServletResponse.SC_BAD_REQUEST, path); + } + catch (Throwable t) { + sendErrorAsXml(httpServletResponse, t, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, path); + } + finally { + MDC.remove("User"); + httpServletRequest.removeAttribute(AUTHENTICATION_TOKEN); + if (authenticatedToken != null) { + ugiManager.removeRequest(authenticatedToken.getPrincipal(), httpServletRequest); + } + } + } + + private AuthenticationToken getAuthenticationTokenFromRequest(HttpServletRequest httpServletRequest) { + Cookie authenticatedCookie = CookieHelper.extract(httpServletRequest); + if (authenticatedCookie != null) { + CookieBasedAuthenticationToken authenticatedToken = (CookieBasedAuthenticationToken) AuthenticationTokenSerDe + .deserialize(authenticatedCookie.getValue(), httpServletRequest.getRemoteAddr(), + cookieSignerVerifier); + MDC.put("User", authenticatedToken.getPrincipal()); + LOGGER.info("Cookie had a valid authentication token. Original authentication method:" + + authenticatedToken.getInitialAuthMethod()); + return authenticatedToken; + } + return null; + } + + /** + * Authenticate the requests for protected resources. + * + * @param httpServletRequest http servlet request + * @return authentication token + * @throws IOException thrown if failed to get authentication token + */ + private AuthenticationToken authenticate(HttpServletRequest httpServletRequest) throws IOException { + AuthenticationProvider supportedProvider = providerFactory.getAuthenticationProvider(httpServletRequest); + AuthenticationToken rawToken = supportedProvider.getAuthenticationToken(httpServletRequest); + AuthenticationToken authenticatedToken = supportedProvider.authenticate(rawToken); + + if (!authenticatedToken.isAuthenticated()) { + throw new AccessDeniedException("Authentication failed for: " + authenticatedToken.getPrincipal()); + } + return authenticatedToken; + } + + private void handleAuthenticationFailure(HttpServletResponse httpServletResponse, Exception e, String path) { + httpServletResponse.addCookie(CookieHelper.createExpiredCookie()); + sendErrorAsXml(httpServletResponse, e, HttpServletResponse.SC_UNAUTHORIZED, path); + } + + private void sendErrorAsXml(HttpServletResponse httpServletResponse, Throwable t, int statusCode, String path) { + LOGGER.error("Error processing Authentication.", t); + try { + HttpExceptionUtil.sendErrorAsXml(httpServletResponse, statusCode, t, path); + } + catch (IOException e) { + LOGGER.error("Error sending failure.", e); + } + } + + @Override + public void destroy() { + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieHelper.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieHelper.java new file mode 100644 index 000000000..9237eccf2 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieHelper.java @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.http.authentication.AuthenticationTokenSerDe; + +/** + * Provide utility functions for Cookie. + * + */ +final class CookieHelper { + private static final String COOKIE_NAME = "Hadoop-HTTP-Auth"; + + private CookieHelper() { + } + + /** + * Create cookie from serialized authenticated token. + * + * @param serializedAuthenticatedToken + * @return cookie + */ + static Cookie create(String serializedAuthenticatedToken) { + Cookie cookie = new Cookie(COOKIE_NAME, serializedAuthenticatedToken); + cookie.setMaxAge((int) AuthenticationTokenSerDe.EXPIRATION_IN_SECONDS); + cookie.setPath("/"); + return cookie; + } + + /** + * Create expired cookie + * + * @return cookie + */ + static Cookie createExpiredCookie() { + Cookie cookie = new Cookie(COOKIE_NAME, "Expired cookie To clear browsers"); + cookie.setMaxAge(0); + cookie.setPath("/"); + return cookie; + } + + /** + * Get cookie from http servlet request + * + * @param httpServletRequest http servlet request + * @return cookie + */ + static Cookie extract(HttpServletRequest httpServletRequest) { + Cookie authenticatedCookie = null; + Cookie[] cookies = httpServletRequest.getCookies(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (COOKIE_NAME.equals(cookie.getName())) { + authenticatedCookie = cookie; + break; + } + } + } + + return authenticatedCookie; + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieSignerVerifier.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieSignerVerifier.java new file mode 100644 index 000000000..0f7df4fb4 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/CookieSignerVerifier.java @@ -0,0 +1,182 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; + +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class CookieSignerVerifier is to provide public/private security mechanism to verify and sign + * cookie. + */ +public class CookieSignerVerifier { + + private static final Logger LOGGER = LoggerFactory.getLogger(CookieSignerVerifier.class.getName()); + private String keyAlgorithm = "RSA"; + private String signatureAlgorithm = "MD5withRSA"; + private KeyFactory keyFactory = null; + private PrivateKey privateKey = null; + private PublicKey publicKey = null; + + public CookieSignerVerifier(Configuration conf) throws Exception { + keyFactory = KeyFactory.getInstance(keyAlgorithm); + + String publicKeyFile = conf.get("cookie.signer.public.key.file"); + String privateKeyFile = conf.get("cookie.signer.private.key.file"); + String certificateFile = conf.get("cookie.signer.certificate.file"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Public key file: " + publicKeyFile); + LOGGER.debug("Private key file: " + privateKeyFile); + LOGGER.debug("Certificate file: " + certificateFile); + } + initializeKeys(publicKeyFile, privateKeyFile, certificateFile); + } + + /** + * Initialize the keys + * + * @param publicKeyFile public key file + * @param privateKeyFile private key file + * @param certificateFile certificate file + * @throws GeneralSecurityException thrown if error + * @throws IOException thrown if error + */ + private void initializeKeys(String publicKeyFile, String privateKeyFile, String certificateFile) + throws GeneralSecurityException, IOException { + + if (publicKeyFile == null && certificateFile == null && privateKeyFile == null) { + LOGGER.info("Creating random public and private keys for cookie signing."); + KeyPair keyPair = KeyPairGenerator.getInstance(keyAlgorithm).generateKeyPair(); + privateKey = keyPair.getPrivate(); + publicKey = keyPair.getPublic(); + } + else if ((publicKeyFile != null || certificateFile != null) && privateKeyFile != null) { + privateKey = getPrivateKey(privateKeyFile); + + if (publicKeyFile != null) { + publicKey = getPublicKey(publicKeyFile); + } + else { + Certificate cert = getCertificate(certificateFile); + publicKey = cert.getPublicKey(); + } + } + else { + throw new IllegalArgumentException("Both public and private key should be configured"); + } + } + + /** + * Get cookie signature + * + * @param data data to be signed + * @return signed data + * @throws GeneralSecurityException + */ + public String getSignature(String data) throws GeneralSecurityException { + Signature signer = Signature.getInstance(signatureAlgorithm); + signer.initSign(privateKey); + signer.update(data.getBytes()); + byte[] signature = signer.sign(); + return Base64.encodeBase64URLSafeString(signature); + } + + /** + * Verify cookie signature + * + * @param data signed data + * @param signature signature + * @return true if verified successfully + * @throws GeneralSecurityException + */ + public boolean verifySignature(String data, String signature) throws GeneralSecurityException { + Signature verifier = Signature.getInstance(signatureAlgorithm); + verifier.initVerify(publicKey); + verifier.update(data.getBytes()); + byte[] signatureBytes = Base64.decodeBase64(signature.getBytes()); + return verifier.verify(signatureBytes); + } + + private PublicKey getPublicKey(String fileName) throws IOException, GeneralSecurityException { + /* + * Generate an RSA key : + * openssl genrsa -rand -des3 -out rsakey.pem 1024 + * Create private key in .der format + * openssl pkcs8 -topk8 -nocrypt -inform PEM -in rsakey.pem -outform DER -out private.der + * Create public key in .der format + * openssl rsa -inform PEM -in rsakey.pem -outform DER -out public.der -pubout + */ + FileInputStream fis = null; + try { + fis = new FileInputStream(fileName); + byte[] keyBytes = new byte[fis.available()]; + fis.read(keyBytes); + X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); + return keyFactory.generatePublic(keySpec); + } + finally { + IOUtils.closeStream(fis); + } + } + + private Certificate getCertificate(String fileName) throws FileNotFoundException, CertificateException { + FileInputStream fis = null; + try { + fis = new FileInputStream(fileName); + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(fis); + return cert; + } + finally { + IOUtils.closeStream(fis); + } + } + + private PrivateKey getPrivateKey(String fileName) throws IOException, InvalidKeySpecException { + FileInputStream fis = null; + try { + fis = new FileInputStream(fileName); + byte[] keyBytes = new byte[fis.available()]; + fis.read(keyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); + return keyFactory.generatePrivate(keySpec); + } + finally { + IOUtils.closeStream(fis); + } + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/EvictorCallback.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/EvictorCallback.java new file mode 100644 index 000000000..8fd763723 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/EvictorCallback.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import org.apache.hadoop.security.UserGroupInformation; + +public interface EvictorCallback { + + public void callback(UserGroupInformation ugi); +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/FileSystemEvictorCallback.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/FileSystemEvictorCallback.java new file mode 100644 index 000000000..ae98a4617 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/FileSystemEvictorCallback.java @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The class FileSystemEvictorCallback is to write information down when eviction of ugi happens. + * + */ +public class FileSystemEvictorCallback implements EvictorCallback { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemEvictorCallback.class); + + @Override + public void callback(final UserGroupInformation ugi) { + try { + FileSystem.closeAllForUGI(ugi); + LOGGER.info("Closed all filesystems for user " + ugi.getShortUserName()); + } + catch (Throwable t) { + LOGGER.warn("Exception while closing filesystem for user " + ugi.getShortUserName(), t); + } + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/ProxyUGICacheManager.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/ProxyUGICacheManager.java new file mode 100644 index 000000000..d6825403d --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/ProxyUGICacheManager.java @@ -0,0 +1,205 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The container class ProxyUGICacheManager is to maintain user ugi entries and service cached ugi without + * recreating new ones. + */ +public class ProxyUGICacheManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(ProxyUGICacheManager.class.getName()); + + private ConcurrentMap userUgiMap; + private final ScheduledThreadPoolExecutor evictorDaemon; + + public ProxyUGICacheManager(long ugiExpiryTimeInMillis, long evictionIntervalInMillis, EvictorCallback callback) { + userUgiMap = new ConcurrentHashMap(); + evictorDaemon = new ScheduledThreadPoolExecutor(1); + CacheEvictor evictor = new CacheEvictor(ugiExpiryTimeInMillis, callback); + if (callback != null) { + evictorDaemon.scheduleWithFixedDelay(evictor, ugiExpiryTimeInMillis, evictionIntervalInMillis, + TimeUnit.MILLISECONDS); + } + } + + /** + * Get ugi from cached map. + * + * @param user user name + * @param request http servlet request + * @return ugi + * @throws IOException thrown if current user is not log in + */ + public UserGroupInformation getUGI(String user, HttpServletRequest request) throws IOException { + CacheEntry entry = userUgiMap.get(user); + if (entry == null) { + // UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, + // UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation + .createProxyUser(user, UserGroupInformation.getCurrentUser()); + // Take care of race condition + CacheEntry oldEntry = userUgiMap.putIfAbsent(user, new CacheEntry(ugi, request)); + if (oldEntry == null) { + LOGGER.info("Creating new proxy ugi for user " + user); + } + else { + oldEntry.addRequest(request); + } + return ugi; + } + entry.addRequest(request); + return entry.getUgi(); + } + + /** + * Get number of user requests + * + * @param user user name + * @return number of user requests + * @throws IOException thrown if error + */ + public int getNumberOfRequestsForUser(String user) throws IOException { + CacheEntry entry = userUgiMap.get(user); + return entry.getNumRequests(); + } + + /** + * Remove request from cache entry + * + * @param user user name + * @param request http servlet request + */ + public void removeRequest(String user, HttpServletRequest request) { + CacheEntry entry = userUgiMap.get(user); + if (entry == null) { + LOGGER.warn("The Cache Manager should have had the ugi for the user " + user); + return; + } + if (!entry.removeRequest(request)) { + LOGGER.warn("The Cache Manager should have had the request " + request.getRequestURI() + "?" + + request.getQueryString() + " from user " + user); + } + } + + /** + * Destroy cached map and stop cron service + */ + public void destroy() { + evictorDaemon.shutdownNow(); + userUgiMap.clear(); + userUgiMap = null; + } + + /** + * Cache entry to store ugi and request information. + * + */ + private static class CacheEntry { + private UserGroupInformation ugi; + private long lastAccessTime; + private ConcurrentMap requests = new ConcurrentHashMap(); + + public CacheEntry(UserGroupInformation ugi, HttpServletRequest request) { + this.ugi = ugi; + addRequest(request); + } + + public UserGroupInformation getUgi() { + return ugi; + } + + public long getLastAccessedTime() { + return lastAccessTime; + } + + public void addRequest(HttpServletRequest request) { + if (request == null) + throw new IllegalArgumentException("HttpServletRequest cannot be null"); + requests.put(request, Boolean.TRUE); + lastAccessTime = System.currentTimeMillis(); + } + + public boolean removeRequest(HttpServletRequest request) { + lastAccessTime = System.currentTimeMillis(); + return requests.remove(request) == null ? false : true; + } + + public boolean hasRequests() { + return requests.size() != 0; + } + + public int getNumRequests() { + return requests.size(); + } + } + + /** + * Cron service to evict old cache entry from map. + * + */ + private class CacheEvictor implements Runnable { + private final long ugiExpiryTimeInMillis; + private final EvictorCallback callback; + + public CacheEvictor(long ugiExpiryTimeInMillis, EvictorCallback callback) { + this.ugiExpiryTimeInMillis = ugiExpiryTimeInMillis; + this.callback = callback; + } + + public void run() { + try { + if (userUgiMap.isEmpty()) + return; + + long currentTime = System.currentTimeMillis(); + LOGGER.info("Checking UGI cache for expired entries"); + for (ConcurrentMap.Entry mapEntry : userUgiMap.entrySet()) { + CacheEntry cacheEntry = mapEntry.getValue(); + long lastAccessed = cacheEntry.getLastAccessedTime(); + long lifeTime = currentTime - lastAccessed; + if (lifeTime < ugiExpiryTimeInMillis) { + continue; + } + + if (!cacheEntry.hasRequests()) { + LOGGER.info("UGI for user " + mapEntry.getKey() + " has expired. Evicting"); + userUgiMap.remove(mapEntry.getKey()); + callback.callback(cacheEntry.getUgi()); + } + else if (lifeTime > TimeUnit.HOURS.toMillis(3)) { + LOGGER.warn("There is a request running for more than 3 hours using ugi: " + mapEntry.getKey()); + } + } + } + catch (Throwable e) { + LOGGER.error("Error while evicting UGI cache", e); + } + } + } + +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/listener/AppAuthApplicationListener.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/listener/AppAuthApplicationListener.java new file mode 100644 index 000000000..8bf46cc4c --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/listener/AppAuthApplicationListener.java @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web.listener; + +import java.util.Date; +import java.util.concurrent.TimeUnit; + +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.web.AuthenticationProcessingFilter; +import org.apache.hadoop.http.authentication.web.CookieSignerVerifier; +import org.apache.hadoop.http.authentication.web.FileSystemEvictorCallback; +import org.apache.hadoop.http.authentication.web.ProxyUGICacheManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The listener to initialize all required configuration and attributes for servlet context. The attributes and + * configuration are used later in filter class AuthenticationProcessingFilter. + *

+ * Oozie currently overrides this listener initializeAuthConfiguration() to create configuration from Oozie server + * configuration. + *

+ * The listener has to be configured in application server web.xml to load in server start time. + */ +public abstract class AppAuthApplicationListener implements ServletContextListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(AppAuthApplicationListener.class); + + private static final String START_TIME = "StartTime"; + + protected ServletContext servletContext = null; + protected Configuration authConfiguration = null; + protected Configuration applicationConfiguration = null; + + protected abstract Configuration initializeApplicationConfiguration(); + + protected abstract void initializeUGI(); + + /* (non-Javadoc) + * @see javax.servlet.ServletContextListener#contextInitialized(javax.servlet.ServletContextEvent) + */ + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + servletContext = servletContextEvent.getServletContext(); + + Date startDate = new Date(); + LOGGER.info("Server Starting at : " + startDate); + servletContext.setAttribute(START_TIME, startDate); + + authConfiguration = initializeAuthConfiguration(); + servletContext.setAttribute(AuthenticationProcessingFilter.AUTH_CONFIGURATION, authConfiguration); + servletContext.setAttribute(AuthenticationProcessingFilter.COOKIE_SIGNER_VERIFIER, + initializeCookieSignerVerifier(authConfiguration)); + + applicationConfiguration = initializeApplicationConfiguration(); + servletContext.setAttribute(AuthenticationProcessingFilter.UGI_CACHE_MANAGER, + initializeUGICacheManager(applicationConfiguration)); + initializeUGI(); + } + + /** + * Initialize authentication configuration. + * + * @return authentication configuration + */ + protected Configuration initializeAuthConfiguration() { + Configuration configuration = new Configuration(false); + configuration.addResource("authentication-conf.xml"); + configuration.addResource("authentication-site.xml"); + return configuration; + } + + /** + * Initialize {@link ProxyUGICacheManager} + * + * @param conf configuration + * @return instance of {@link ProxyUGICacheManager} + */ + protected ProxyUGICacheManager initializeUGICacheManager(Configuration conf) { + long ugiExpiryTimeInMillis = conf.getLong("ugi.expirytime.in.millis", TimeUnit.MINUTES.toMillis(10)); + long evictionIntervalInMillis = conf.getLong("ugi.evictioninterval.in.millis", TimeUnit.MINUTES.toMillis(5)); + + ProxyUGICacheManager cacheManager = new ProxyUGICacheManager(ugiExpiryTimeInMillis, evictionIntervalInMillis, + new FileSystemEvictorCallback()); + return cacheManager; + } + + /** + * Initialize {@link CookieSignerVerifier} + * + * @param conf configuration + * @return instance of {@link CookieSignerVerifier} + */ + protected CookieSignerVerifier initializeCookieSignerVerifier(Configuration conf) { + try { + return new CookieSignerVerifier(conf); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + /* (non-Javadoc) + * @see javax.servlet.ServletContextListener#contextDestroyed(javax.servlet.ServletContextEvent) + */ + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + servletContext.removeAttribute(START_TIME); + servletContext.removeAttribute(AuthenticationProcessingFilter.COOKIE_SIGNER_VERIFIER); + servletContext.removeAttribute(AuthenticationProcessingFilter.AUTH_CONFIGURATION); + servletContext.removeAttribute(AuthenticationProcessingFilter.UGI_CACHE_MANAGER); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/authentication/web/util/Assert.java b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/util/Assert.java new file mode 100644 index 000000000..822d95381 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/authentication/web/util/Assert.java @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.web.util; + +public class Assert { + private Assert() { + } + + /** + * Check not null + * + * @param object object to check + * @param message error message + */ + public static void notNull(Object object, String message) { + if (object == null) { + throw new IllegalArgumentException(message); + } + } + + /** + * Check if true + * + * @param condition boolean value + * @param message error message + */ + public static void isTrue(boolean condition, String message) { + if (!condition) { + throw new IllegalArgumentException(message); + } + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/exception/HttpExceptionUtil.java b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpExceptionUtil.java new file mode 100644 index 000000000..f40f59ad0 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpExceptionUtil.java @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.exception; + +import org.znerd.xmlenc.XMLOutputter; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.StringWriter; + +/** + * Provide utility functions for http errors. + * + */ +public class HttpExceptionUtil { + + /** + * Output error xml + * + * @param t throwable + * @param path url + * @param doc the outputter + * @throws IOException + */ + public static void writeErrorXml(Throwable t, String path, XMLOutputter doc) throws IOException { + doc.startTag(HttpRemoteException.class.getSimpleName()); + if (path == null) + path = "/"; + doc.attribute("path", path); + + if (t instanceof HttpRemoteException) { + doc.attribute("class", ((HttpRemoteException) t).getClassName()); + } + else { + doc.attribute("class", t.getClass().getName()); + } + + String msg = t.getLocalizedMessage(); + if (msg == null) + msg = ""; + + Throwable cause = t.getCause(); + if (cause != null) { + msg += " " + cause.getClass().getName() + ":" + cause.getMessage(); + } + + doc.attribute("message", msg); + doc.endTag(); + } + + /** + * Response error in xml + * + * @param response http servlet response + * @param errorCode error code + * @param t throwable + * @param path the url + * @throws IOException thrown if error + */ + public static void sendErrorAsXml(HttpServletResponse response, int errorCode, Throwable t, String path) + throws IOException { + StringWriter writer = new StringWriter(); + XMLOutputter doc = new XMLOutputter(writer, "UTF-8"); + doc.declaration(); + writeErrorXml(t, path, doc); + doc.endDocument(); + response.sendError(errorCode, writer.toString()); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRemoteException.java b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRemoteException.java new file mode 100644 index 000000000..63e681af0 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRemoteException.java @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.exception; + +import java.lang.reflect.Constructor; + +import org.xml.sax.Attributes; + +public class HttpRemoteException extends Exception { + + private static final long serialVersionUID = 1L; + + private String className; + + public HttpRemoteException(String className, String msg) { + super(msg); + this.className = className; + } + + public String getClassName() { + return className; + } + + /** + * Instantiate and return the exception wrapped up by this remote exception. + * This unwraps any Throwable that has a constructor taking a + * String as a parameter. Otherwise it returns this. + * + * @return Throwable + */ + public Exception unwrapRemoteException() { + try { + Class realClass = Class.forName(getClassName()); + return instantiateException(realClass.asSubclass(Exception.class)); + } + catch (Throwable ignore) { + // cannot instantiate the original exception, just return this + } + return this; + } + + private Exception instantiateException(Class cls) throws Exception { + Constructor cn = cls.getConstructor(String.class); + cn.setAccessible(true); + String firstLine = this.getMessage(); + Exception ex = cn.newInstance(firstLine); + ex.initCause(this); + return ex; + } + + /** Create RemoteException from attributes */ + public static HttpRemoteException valueOf(Attributes attrs) { + return new HttpRemoteException(attrs.getValue("class"), attrs.getValue("message")); + } +} diff --git a/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRunTimeException.java b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRunTimeException.java new file mode 100644 index 000000000..a6ec33422 --- /dev/null +++ b/authentication/src/main/java/org/apache/hadoop/http/exception/HttpRunTimeException.java @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.exception; + +public class HttpRunTimeException extends RuntimeException { + + private static final long serialVersionUID = 1L; + private final int errorCode; + + public HttpRunTimeException(int errorCode) { + super(); + this.errorCode = errorCode; + } + + public HttpRunTimeException(String message, Throwable cause, int errorCode) { + super(message, cause); + this.errorCode = errorCode; + } + + public HttpRunTimeException(Throwable cause, int errorCode) { + super(cause); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } + +} diff --git a/authentication/src/test/java/org/apache/hadoop/http/authentication/test/CookieSignerVerifierTest.java b/authentication/src/test/java/org/apache/hadoop/http/authentication/test/CookieSignerVerifierTest.java new file mode 100644 index 000000000..86f77f420 --- /dev/null +++ b/authentication/src/test/java/org/apache/hadoop/http/authentication/test/CookieSignerVerifierTest.java @@ -0,0 +1,69 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.test; + +import java.net.URL; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.web.CookieSignerVerifier; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CookieSignerVerifierTest { + + private static final String text = "Plain text 24xfse@%#&#$%$="; + private static final String sameLengthText = "Plain text 23xfse@%#&#$%$="; + + @Test + public void testCookieSigningWithRandomGeneratedKey() throws Exception { + Configuration conf = new Configuration(false); + CookieSignerVerifier cookieSignerVerifier = new CookieSignerVerifier(conf); + String signature = cookieSignerVerifier.getSignature(text); + Assert.assertTrue(cookieSignerVerifier.verifySignature(text, signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(text + ",", signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(sameLengthText + ",", signature)); + } + + @Test + public void testCookieSigningWithPublicAndPrivateKeyFromFile() throws Exception { + Configuration conf = new Configuration(false); + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + URL publicKey = classLoader.getResource("TestKeyPair1_public.der"); + URL privateKey = classLoader.getResource("TestKeyPair1_private.der"); + conf.set("cookie.signer.private.key.file", privateKey.getFile()); + conf.set("cookie.signer.public.key.file", publicKey.getFile()); + CookieSignerVerifier cookieSignerVerifier = new CookieSignerVerifier(conf); + String signature = cookieSignerVerifier.getSignature(text); + Assert.assertTrue(cookieSignerVerifier.verifySignature(text, signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(text + ",", signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(sameLengthText + ",", signature)); + } + + @Test + public void testCookieSigningWithPublicKeyInCertificate() throws Exception { + Configuration conf = new Configuration(false); + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + URL certifcate = classLoader.getResource("TestKeyPair2_cert.der"); + URL privateKey = classLoader.getResource("TestKeyPair2_private.der"); + conf.set("cookie.signer.certificate.file", certifcate.getFile()); + conf.set("cookie.signer.private.key.file", privateKey.getFile()); + CookieSignerVerifier cookieSignerVerifier = new CookieSignerVerifier(conf); + String signature = cookieSignerVerifier.getSignature(text); + Assert.assertTrue(cookieSignerVerifier.verifySignature(text, signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(text + ",", signature)); + Assert.assertFalse(cookieSignerVerifier.verifySignature(sameLengthText + ",", signature)); + } + +} diff --git a/authentication/src/test/java/org/apache/hadoop/http/authentication/test/ProxyUGICacheManagerTest.java b/authentication/src/test/java/org/apache/hadoop/http/authentication/test/ProxyUGICacheManagerTest.java new file mode 100644 index 000000000..23162d476 --- /dev/null +++ b/authentication/src/test/java/org/apache/hadoop/http/authentication/test/ProxyUGICacheManagerTest.java @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.http.authentication.test; + +import java.security.PrivilegedExceptionAction; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.http.authentication.web.FileSystemEvictorCallback; +import org.apache.hadoop.http.authentication.web.ProxyUGICacheManager; +import org.apache.hadoop.security.UserGroupInformation; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ProxyUGICacheManagerTest { + + @Test + public void testEviction() throws Exception { + Configuration conf = new Configuration(true); + FileSystemEvictorCallback callback = new FileSystemEvictorCallback(); + ProxyUGICacheManager manager = new ProxyUGICacheManager(20, 50, callback); + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + UserGroupInformation ugi = manager.getUGI("nobody", request); + FileSystem fs = getFileSystem(ugi, conf); + FileSystem fs1 = getFileSystem(ugi, conf); + if (fs != fs1) + Assert.fail(); + manager.removeRequest("nobody", request); + Thread.sleep(90); + FileSystem fs2 = getFileSystem(ugi, conf); + if (fs == fs2) + Assert.fail(); + manager.destroy(); + } + + private FileSystem getFileSystem(UserGroupInformation ugi, final Configuration conf) throws Exception { + return ugi.doAs(new PrivilegedExceptionAction() { + @Override + public FileSystem run() throws Exception { + FileSystem fs = FileSystem.get(conf); + return fs; + } + }); + } + +} diff --git a/authentication/src/test/resources/TestKeyPair1_private.der b/authentication/src/test/resources/TestKeyPair1_private.der new file mode 100644 index 000000000..ae9bc9653 Binary files /dev/null and b/authentication/src/test/resources/TestKeyPair1_private.der differ diff --git a/authentication/src/test/resources/TestKeyPair1_public.der b/authentication/src/test/resources/TestKeyPair1_public.der new file mode 100644 index 000000000..d65dd5f31 Binary files /dev/null and b/authentication/src/test/resources/TestKeyPair1_public.der differ diff --git a/authentication/src/test/resources/TestKeyPair2_cert.der b/authentication/src/test/resources/TestKeyPair2_cert.der new file mode 100644 index 000000000..4d8624464 Binary files /dev/null and b/authentication/src/test/resources/TestKeyPair2_cert.der differ diff --git a/authentication/src/test/resources/TestKeyPair2_private.der b/authentication/src/test/resources/TestKeyPair2_private.der new file mode 100644 index 000000000..9eb123a83 Binary files /dev/null and b/authentication/src/test/resources/TestKeyPair2_private.der differ diff --git a/client/pom.xml b/client/pom.xml index cffe29fe0..ccf902fd9 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -58,6 +58,16 @@ persistence-api provided + + com.yahoo.oozie + oozie-authentication + compile + + + org.slf4j + slf4j-log4j12 + compile + diff --git a/client/src/main/conf/client-site.xml b/client/src/main/conf/client-site.xml new file mode 100644 index 000000000..3afd61940 --- /dev/null +++ b/client/src/main/conf/client-site.xml @@ -0,0 +1,29 @@ + + + + + + + + oozie.auth.map + simple=org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator + + List of Oozie client authenticator (separated by commas). + The format is key=classname and key will be used in command line for -auth options. + + + + \ No newline at end of file diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index 183de408f..2ad13c1a3 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -16,12 +16,14 @@ import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -111,10 +113,17 @@ public class OozieCLI { public static final String VERBOSE_OPTION = "verbose"; public static final String VERBOSE_DELIMITER = "\t"; + public static final String ENV_CLIENT_SITE = "CLIENT_SITE"; + public static final String CLIENT_SITE_OPTION = "clientsite"; + public static final String AUTH_SITE_KEY = "oozie.auth.map"; + public static final String AUTH_OPTION = "auth"; + public static final String PIGFILE_OPTION = "file"; private static final String[] OOZIE_HELP = { "the env variable '" + ENV_OOZIE_URL + "' is used as default value for the '-" + OOZIE_OPTION + "' option", + "the env variable '" + ENV_CLIENT_SITE + "' is used as default value for the '-" + CLIENT_SITE_OPTION + + "' option", "custom headers for Oozie web services can be specified using '-D" + WS_HEADER_PREFIX + "NAME=VALUE'" }; private static final String RULER; @@ -133,8 +142,7 @@ public class OozieCLI { /** * Entry point for the Oozie CLI when invoked from the command line. *

- * Upon completion this method exits the JVM with '0' (success) or '-1' - * (failure). + * Upon completion this method exits the JVM with '0' (success) or '-1' (failure). * * @param args options and arguments for the Oozie CLI. */ @@ -165,8 +173,13 @@ protected Options createAdminOptions() { Option status = new Option(STATUS_OPTION, false, "show the current system status"); Option version = new Option(VERSION_OPTION, false, "show Oozie server build version"); Option queuedump = new Option(QUEUE_DUMP_OPTION, false, "show Oozie server queue elements"); + Option site = new Option(CLIENT_SITE_OPTION, true, "set Oozie client site path"); + Option auth = new Option(AUTH_OPTION, true, + "select authentication type (default 'simple', requires '-clientsite' or env 'CLIENT_SITE')"); Options adminOptions = new Options(); adminOptions.addOption(oozie); + adminOptions.addOption(site); + adminOptions.addOption(auth); OptionGroup group = new OptionGroup(); group.addOption(system_mode); group.addOption(status); @@ -181,8 +194,7 @@ protected Options createJobOptions() { Option config = new Option(CONFIG_OPTION, true, "job configuration file '.xml' or '.properties'"); Option submit = new Option(SUBMIT_OPTION, false, "submit a job"); Option run = new Option(RUN_OPTION, false, "run a job"); - Option rerun = new Option(RERUN_OPTION, true, - "rerun a job (coordinator requires -action or -date)"); + Option rerun = new Option(RERUN_OPTION, true, "rerun a job (coordinator requires -action or -date)"); Option dryrun = new Option(DRYRUN_OPTION, false, "Supported in Oozie-2.0 or later versions ONLY - dryrun or test run a coordinator job, job is not queued"); Option start = new Option(START_OPTION, true, "start a job"); @@ -190,7 +202,8 @@ protected Options createJobOptions() { Option resume = new Option(RESUME_OPTION, true, "resume a job"); Option kill = new Option(KILL_OPTION, true, "kill a job"); Option change = new Option(CHANGE_OPTION, true, "change a coordinator job"); - Option changeValue = new Option(CHANGE_VALUE_OPTION, true, "new endtime/concurrency/pausetime value for changing a coordinator job"); + Option changeValue = new Option(CHANGE_VALUE_OPTION, true, + "new endtime/concurrency/pausetime value for changing a coordinator job"); Option info = new Option(INFO_OPTION, true, "info of a job"); Option offset = new Option(OFFSET_OPTION, true, "job info offset of actions (default '1', requires -info)"); Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)"); @@ -204,8 +217,11 @@ protected Options createJobOptions() { "re-materialize the coordinator rerun actions (requires -rerun)"); Option rerun_nocleanup = new Option(RERUN_NOCLEANUP_OPTION, false, "do not clean up output-events of the coordiantor rerun actions (requires -rerun)"); - Option property = OptionBuilder.withArgName( "property=value" ).hasArgs(2) - .withValueSeparator().withDescription( "set/override value for given property" ).create( "D" ); + Option site = new Option(CLIENT_SITE_OPTION, true, "set Oozie client site path"); + Option auth = new Option(AUTH_OPTION, true, + "select authentication type (default 'simple', requires '-clientsite' or env 'CLIENT_SITE')"); + Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( + "set/override value for given property").create("D"); OptionGroup actions = new OptionGroup(); actions.addOption(submit); @@ -234,6 +250,8 @@ protected Options createJobOptions() { jobOptions.addOption(rerun_date); jobOptions.addOption(rerun_refresh); jobOptions.addOption(rerun_nocleanup); + jobOptions.addOption(site); + jobOptions.addOption(auth); jobOptions.addOptionGroup(actions); return jobOptions; } @@ -241,11 +259,15 @@ protected Options createJobOptions() { protected Options createJobsOptions() { Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); Option start = new Option(OFFSET_OPTION, true, "jobs offset (default '1')"); - Option jobtype = new Option(JOBTYPE_OPTION, true, "job type ('Supported in Oozie-2.0 or later versions ONLY - coordinator' or 'wf' (default))"); + Option jobtype = new Option(JOBTYPE_OPTION, true, + "job type ('Supported in Oozie-2.0 or later versions ONLY - coordinator' or 'wf' (default))"); Option len = new Option(LEN_OPTION, true, "number of jobs (default '100')"); Option filter = new Option(FILTER_OPTION, true, "user=;name=;group=;status=;..."); Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (default GMT)"); Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode"); + Option site = new Option(CLIENT_SITE_OPTION, true, "set Oozie client site path"); + Option auth = new Option(AUTH_OPTION, true, + "select authentication type (default 'simple', requires '-clientsite' or env 'CLIENT_SITE')"); start.setType(Integer.class); len.setType(Integer.class); Options jobsOptions = new Options(); @@ -253,10 +275,11 @@ protected Options createJobsOptions() { jobsOptions.addOption(localtime); jobsOptions.addOption(start); jobsOptions.addOption(len); - jobsOptions.addOption(oozie); jobsOptions.addOption(filter); jobsOptions.addOption(jobtype); jobsOptions.addOption(verbose); + jobsOptions.addOption(site); + jobsOptions.addOption(auth); return jobsOptions; } @@ -264,12 +287,17 @@ protected Options createSlaOptions() { Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')"); Option len = new Option(LEN_OPTION, true, "number of results (default '100')"); + Option site = new Option(CLIENT_SITE_OPTION, true, "set Oozie client site path"); + Option auth = new Option(AUTH_OPTION, true, + "select authentication type (default 'simple', requires '-clientsite' or env 'CLIENT_SITE')"); start.setType(Integer.class); len.setType(Integer.class); Options slaOptions = new Options(); slaOptions.addOption(start); slaOptions.addOption(len); slaOptions.addOption(oozie); + slaOptions.addOption(site); + slaOptions.addOption(auth); return slaOptions; } @@ -277,13 +305,18 @@ protected Options createPigOptions() { Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL"); Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'"); Option pigFile = new Option(PIGFILE_OPTION, true, "Pig script"); - Option property = OptionBuilder.withArgName( "property=value" ).hasArgs(2) - .withValueSeparator().withDescription( "set/override value for given property" ).create( "D" ); + Option site = new Option(CLIENT_SITE_OPTION, true, "set Oozie client site path"); + Option auth = new Option(AUTH_OPTION, true, + "select authentication type (default 'simple', requires '-clientsite' or env 'CLIENT_SITE')"); + Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription( + "set/override value for given property").create("D"); Options pigOptions = new Options(); pigOptions.addOption(oozie); pigOptions.addOption(config); pigOptions.addOption(property); pigOptions.addOption(pigFile); + pigOptions.addOption(site); + pigOptions.addOption(auth); return pigOptions; } @@ -312,7 +345,7 @@ public synchronized int run(String[] args) { parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true); parser.addCommand(SLA_CMD, "", "sla operations (Supported in Oozie-2.0 or later)", createSlaOptions(), false); parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig", - createPigOptions(), true); + createPigOptions(), true); try { CLIParser.Command command = parser.parse(args); @@ -372,6 +405,24 @@ protected String getOozieUrl(CommandLine commandLine) { return url; } + /** + * Return client site path + * + * @param commandLine + * @return client site path + */ + protected String getClientSiteDir(CommandLine commandLine) { + String clientSitePath = commandLine.getOptionValue(CLIENT_SITE_OPTION); + if (clientSitePath == null) { + clientSitePath = System.getenv(ENV_CLIENT_SITE); + if (clientSitePath == null) { + throw new IllegalArgumentException( + "Oozie client site xml is not available neither in command option '-clientsite' or 'CLIENT_SITE' in the environment"); + } + } + return clientSitePath; + } + // Canibalized from Hadoop Configuration.loadResource(). private Properties parse(InputStream is, Properties conf) throws IOException { try { @@ -486,33 +537,89 @@ private void addHeader(OozieClient wc) { } } + private void addAuthClass(OozieClient wc, CommandLine commandLine) throws OozieCLIException { + String authType = commandLine.getOptionValue(AUTH_OPTION); + if (authType == null) { + //use default simple authenticator + return; + } + Properties conf = new Properties(); + String clientSitePath = getClientSiteDir(commandLine); + if (clientSitePath == null) { + throw new OozieCLIException("requires '-clientsite' or env 'CLIENT_SITE' for -auth option"); + } + File sitePath = new File(clientSitePath); + if (sitePath.isDirectory()) { + sitePath = new File(sitePath, "client-site.xml"); + } + + if (!sitePath.exists() || !sitePath.getName().endsWith(".xml")) { + throw new OozieCLIException("client-site.xml or specified client site xml is missing"); + } + + try { + parse(new FileInputStream(sitePath.getAbsolutePath()), conf); + } + catch (FileNotFoundException e) { + throw new OozieCLIException("exception in oozie cli.", e); + } + catch (IOException e) { + throw new OozieCLIException("exception in oozie cli.", e); + } + + Map authMap = new HashMap(); + String confValue = (String) conf.get(AUTH_SITE_KEY); + if (confValue == null || confValue.isEmpty()) { + throw new OozieCLIException("'" + AUTH_SITE_KEY + "' value is missing from client site xml"); + } + confValue = confValue.trim(); + String[] values = confValue.split(","); + + for (String value : values) { + String[] arr = value.trim().split("="); + if (arr.length != 2) { + throw new OozieCLIException("format of '" + AUTH_SITE_KEY + "' is wrong in client site xml : " + value); + } + String key = arr[0]; + String className = arr[1]; + authMap.put(key, className); + } + if (authMap.get(authType) != null) { + wc.setAuthClass(authMap.get(authType)); + } else { + throw new OozieCLIException("specified authentication type is missing from client site xml"); + } + } + /** - * Create a OozieClient.

It injects any '-Dheader:' as header to the the {@link - * org.apache.oozie.client.OozieClient}. + * Create a OozieClient. + *

+ * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}. * * @param commandLine the parsed command line options. * @return a pre configured eXtended workflow client. - * @throws OozieCLIException thrown if the OozieClient could not be - * configured. + * @throws OozieCLIException thrown if the OozieClient could not be configured. */ protected OozieClient createOozieClient(CommandLine commandLine) throws OozieCLIException { OozieClient wc = new OozieClient(getOozieUrl(commandLine)); addHeader(wc); + addAuthClass(wc, commandLine); return wc; } /** - * Create a XOozieClient.

It injects any '-Dheader:' as header to the the {@link - * org.apache.oozie.client.OozieClient}. + * Create a XOozieClient. + *

+ * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}. * * @param commandLine the parsed command line options. * @return a pre configured eXtended workflow client. - * @throws OozieCLIException thrown if the XOozieClient could not be - * configured. + * @throws OozieCLIException thrown if the XOozieClient could not be configured. */ protected XOozieClient createXOozieClient(CommandLine commandLine) throws OozieCLIException { XOozieClient wc = new XOozieClient(getOozieUrl(commandLine)); addHeader(wc); + addAuthClass(wc, commandLine); return wc; } @@ -567,7 +674,8 @@ else if (options.contains(RUN_OPTION)) { else if (options.contains(RERUN_OPTION)) { if (commandLine.getOptionValue(RERUN_OPTION).contains("-W")) { wc.reRun(commandLine.getOptionValue(RERUN_OPTION), getConfiguration(commandLine)); - } else { + } + else { String coordJobId = commandLine.getOptionValue(RERUN_OPTION); String scope = null; String rerunType = null; @@ -580,7 +688,8 @@ else if (options.contains(RERUN_OPTION)) { if (options.contains(RERUN_DATE_OPTION)) { rerunType = RestConstants.JOB_COORD_RERUN_DATE; scope = commandLine.getOptionValue(RERUN_DATE_OPTION); - } else if (options.contains(RERUN_ACTION_OPTION)){ + } + else if (options.contains(RERUN_ACTION_OPTION)) { rerunType = RestConstants.JOB_COORD_RERUN_ACTION; scope = commandLine.getOptionValue(RERUN_ACTION_OPTION); } @@ -653,7 +762,8 @@ private void printCoordJob(CoordinatorJob coordJob, boolean localtime, boolean v + VERBOSE_DELIMITER + "Error Code" + VERBOSE_DELIMITER + "Error Message" + VERBOSE_DELIMITER + "External ID" + VERBOSE_DELIMITER + "External Status" + VERBOSE_DELIMITER + "Job ID" + VERBOSE_DELIMITER + "Tracker URI" + VERBOSE_DELIMITER + "Created" + VERBOSE_DELIMITER - + "Nominal Time" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified" + VERBOSE_DELIMITER + "Missing Dependencies"); + + "Nominal Time" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified" + + VERBOSE_DELIMITER + "Missing Dependencies"); System.out.println(RULER); for (CoordinatorAction action : actions) { @@ -663,8 +773,9 @@ private void printCoordJob(CoordinatorJob coordJob, boolean localtime, boolean v + VERBOSE_DELIMITER + maskIfNull(action.getExternalId()) + VERBOSE_DELIMITER + maskIfNull(action.getExternalStatus()) + VERBOSE_DELIMITER + maskIfNull(action.getJobId()) + VERBOSE_DELIMITER + maskIfNull(action.getTrackerUri()) + VERBOSE_DELIMITER - + maskDate(action.getCreatedTime(), localtime) + VERBOSE_DELIMITER + maskDate(action.getNominalTime(), localtime) - + action.getStatus() + VERBOSE_DELIMITER + maskDate(action.getLastModifiedTime(), localtime) + VERBOSE_DELIMITER + + maskDate(action.getCreatedTime(), localtime) + VERBOSE_DELIMITER + + maskDate(action.getNominalTime(), localtime) + action.getStatus() + VERBOSE_DELIMITER + + maskDate(action.getLastModifiedTime(), localtime) + VERBOSE_DELIMITER + maskIfNull(action.getMissingDependencies())); System.out.println(RULER); @@ -675,11 +786,10 @@ private void printCoordJob(CoordinatorJob coordJob, boolean localtime, boolean v "Nominal Time", "Last Mod")); for (CoordinatorAction action : actions) { - System.out.println(String - .format(COORD_ACTION_FORMATTER, maskIfNull(action.getId()), action.getStatus(), - maskIfNull(action.getExternalId()), maskIfNull(action.getErrorCode()), maskDate(action - .getCreatedTime(), localtime), maskDate(action.getNominalTime(), localtime), - maskDate(action.getLastModifiedTime(), localtime))); + System.out.println(String.format(COORD_ACTION_FORMATTER, maskIfNull(action.getId()), + action.getStatus(), maskIfNull(action.getExternalId()), maskIfNull(action.getErrorCode()), + maskDate(action.getCreatedTime(), localtime), maskDate(action.getNominalTime(), localtime), + maskDate(action.getLastModifiedTime(), localtime))); System.out.println(RULER); } @@ -916,27 +1026,27 @@ private void adminCommand(CommandLine commandLine) throws OozieCLIException { System.out.println("Oozie server build version: " + wc.getServerBuildVersion()); } else if (options.contains(SYSTEM_MODE_OPTION)) { - String systemModeOption = commandLine.getOptionValue(SYSTEM_MODE_OPTION).toUpperCase(); - try { - status = SYSTEM_MODE.valueOf(systemModeOption); - } - catch (Exception e) { - throw new OozieCLIException("Invalid input provided for option: " + SYSTEM_MODE_OPTION - + " value given :" + systemModeOption - + " Expected values are: NORMAL/NOWEBSERVICE/SAFEMODE "); - } - wc.setSystemMode(status); - System.out.println("System mode: " + status); + String systemModeOption = commandLine.getOptionValue(SYSTEM_MODE_OPTION).toUpperCase(); + try { + status = SYSTEM_MODE.valueOf(systemModeOption); + } + catch (Exception e) { + throw new OozieCLIException("Invalid input provided for option: " + SYSTEM_MODE_OPTION + + " value given :" + systemModeOption + + " Expected values are: NORMAL/NOWEBSERVICE/SAFEMODE "); + } + wc.setSystemMode(status); + System.out.println("System mode: " + status); } else if (options.contains(STATUS_OPTION)) { - status = wc.getSystemMode(); - System.out.println("System mode: " + status); + status = wc.getSystemMode(); + System.out.println("System mode: " + status); } else if (options.contains(QUEUE_DUMP_OPTION)) { System.out.println("[Server Queue Dump]:"); List list = wc.getQueueDump(); if (list != null && list.size() != 0) { - for (String str: list) { + for (String str : list) { System.out.println(str); } } diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 9420806e1..c290e5ac1 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -51,6 +51,9 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.apache.hadoop.http.authentication.client.HttpAuthenticator; +import org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator; + /** * Client API to submit and manage Oozie workflow jobs against an Oozie intance. *

@@ -114,7 +117,7 @@ public class OozieClient { public static final String CHANGE_VALUE_PAUSETIME = "pausetime"; public static final String CHANGE_VALUE_CONCURRENCY = "concurrency"; - + public static final String LIBPATH = "oozie.libpath"; public static final String USE_SYSTEM_LIBPATH = "oozie.use.system.libpath"; @@ -127,8 +130,7 @@ public static enum SYSTEM_MODE { private String protocolUrl; private boolean validatedVersion = false; private Map headers = new HashMap(); - - + private String authClass = "org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator"; protected OozieClient() { } @@ -323,9 +325,51 @@ protected HttpURLConnection createConnection(URL url, String method) throws IOEx for (Map.Entry header : headers.entrySet()) { conn.setRequestProperty(header.getKey(), header.getValue()); } + performAuthentication(conn); return conn; } + /** + * Authenticate the connection by specified authenticator. + * + * @param connection + * @throws OozieClientException + */ + private void performAuthentication(HttpURLConnection connection) throws OozieClientException { + Map conf = new HashMap(); + try { + getHttpAuthenticator().authenticate(conf, connection); + } + catch (IOException ex) { + throw new OozieClientException(OozieClientException.IO_ERROR, ex); + } + } + + /** + * Get user specified Authenticator + * + * @return HttpAuthenticator + * @throws OozieClientException + */ + private HttpAuthenticator getHttpAuthenticator() throws OozieClientException { + HttpAuthenticator authenticator = null; + if (authClass == null || authClass.isEmpty()) { + //use simple authentication instead + authenticator = new SimpleAuthenticator(); + } + + try { + Class klass = Thread.currentThread().getContextClassLoader().loadClass(authClass); + authenticator = (HttpAuthenticator) klass.newInstance(); + } + catch (Exception ex) { + throw new OozieClientException("Could not instantiate HttpAuthenticator [" + authClass + "], " + + ex.getMessage(), ex); + } + + return authenticator; + } + protected abstract class ClientCallable implements Callable { private String method; private String collection; @@ -1195,4 +1239,22 @@ public static T notNull(T obj, String name) { return obj; } + /** + * Set auth class name + * + * @param authClass the authClass to set + */ + public void setAuthClass(String authClass) { + this.authClass = authClass; + } + + /** + * Retrun auth class name + * + * @return the authClass + */ + public String getAuthClass() { + return authClass; + } + } diff --git a/core/pom.xml b/core/pom.xml index 1bd7eeb7c..fd563e37b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -75,12 +75,25 @@ provided + + com.yahoo.oozie + oozie-authentication + compile + + + + org.slf4j + slf4j-log4j12 + compile + + com.yahoo.hadoop hadoop-core provided + com.yahoo.hadoop @@ -94,12 +107,6 @@ provided - - org.slf4j - slf4j-log4j12 - test - - com.googlecode.json-simple json-simple diff --git a/core/src/main/conf/oozie-log4j.properties b/core/src/main/conf/oozie-log4j.properties index 1d9f8ea97..ea25619f5 100644 --- a/core/src/main/conf/oozie-log4j.properties +++ b/core/src/main/conf/oozie-log4j.properties @@ -56,6 +56,6 @@ log4j.logger.oozieops=DEBUG, oozieops log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation log4j.logger.oozieaudit=ALL, oozieaudit log4j.logger.org.apache.oozie=DEBUG, oozie -log4j.logger.org.apache.hadoop=WARN, oozie +log4j.logger.org.apache.hadoop=INFO, oozie log4j.logger.org.mortbay=WARN, oozie log4j.logger.org.hsqldb=WARN, oozie diff --git a/core/src/main/conf/oozie-site.xml b/core/src/main/conf/oozie-site.xml index c0544376a..bbb46662d 100644 --- a/core/src/main/conf/oozie-site.xml +++ b/core/src/main/conf/oozie-site.xml @@ -227,6 +227,15 @@ library path are used. + + + + authentication.providers + org.apache.hadoop.http.authentication.server.simple.SimpleAuthenticationHeaderProvider + + List of Oozie server authentication providers (separated by commas). + + diff --git a/core/src/main/java/org/apache/oozie/filter/OozieAuthFilter.java b/core/src/main/java/org/apache/oozie/filter/OozieAuthFilter.java new file mode 100644 index 000000000..c1bebd646 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/filter/OozieAuthFilter.java @@ -0,0 +1,73 @@ +/** + * Copyright (c) 2010 Yahoo! Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.oozie.filter; + +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import org.apache.oozie.service.AuthorizationService; +import org.apache.oozie.service.Services; +import org.apache.oozie.servlet.JsonRestServlet; + +/** + * The filter checks for whether Oozie has security turned on during + * initialization. In case security is turned on, the username is read from request + * and set as oozie.user.name for later servlets + */ +public class OozieAuthFilter implements Filter { + + private static boolean securityEnabled; + + /** + * Initializes the Filter. Reads the username from the request and set it as oozie.user.name + */ + public void init(FilterConfig config) throws ServletException { + securityEnabled = Services.get().getConf().getBoolean(AuthorizationService.CONF_SECURITY_ENABLED, true); + } + + public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, + ServletException { + if (securityEnabled) { + HttpServletRequest request = (HttpServletRequest) req; + String userName = request.getUserPrincipal().getName(); + setUserName(request, userName); + } + chain.doFilter(req, res); + } + + /** + * Take care of cleanup. NOP for now. + */ + public void destroy() { + } + + /** + * Sets the user name to be used later in the chain or by servlets. + * + * @param request the request object + * @param userId the user name to set + */ + private void setUserName(HttpServletRequest request, String userId) { + request.setAttribute(JsonRestServlet.USER_NAME, userId); + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/oozie/servlet/ServicesLoader.java b/core/src/main/java/org/apache/oozie/servlet/ServicesLoader.java index d72b09749..67b0881af 100644 --- a/core/src/main/java/org/apache/oozie/servlet/ServicesLoader.java +++ b/core/src/main/java/org/apache/oozie/servlet/ServicesLoader.java @@ -14,6 +14,8 @@ */ package org.apache.oozie.servlet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.web.listener.AppAuthApplicationListener; import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; @@ -23,7 +25,7 @@ /** * Webapp context listener that initializes Oozie {@link Services}. */ -public class ServicesLoader implements ServletContextListener { +public class ServicesLoader extends AppAuthApplicationListener { private static Services services; /** @@ -35,6 +37,7 @@ public void contextInitialized(ServletContextEvent event) { try { services = new Services(); services.init(); + super.contextInitialized(event); } catch (ServiceException ex) { throw new RuntimeException(ex); @@ -50,4 +53,21 @@ public void contextDestroyed(ServletContextEvent event) { services.destroy(); } + @Override + protected Configuration initializeAuthConfiguration() { + Configuration configuration = services.getConf(); + return configuration; + } + + @Override + protected Configuration initializeApplicationConfiguration() { + Configuration configuration = services.getConf(); + return configuration; + } + + @Override + protected void initializeUGI() { + // no-op + } + } diff --git a/core/src/main/java/org/apache/oozie/test/EmbeddedServletContainer.java b/core/src/main/java/org/apache/oozie/test/EmbeddedServletContainer.java index 8c3d76363..8fa1bbf68 100644 --- a/core/src/main/java/org/apache/oozie/test/EmbeddedServletContainer.java +++ b/core/src/main/java/org/apache/oozie/test/EmbeddedServletContainer.java @@ -4,17 +4,18 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. See accompanying LICENSE file. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. */ package org.apache.oozie.test; import org.mortbay.jetty.Server; +import org.mortbay.jetty.servlet.FilterHolder; import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.servlet.Context; @@ -22,8 +23,11 @@ import java.net.ServerSocket; /** - * An embedded servlet container for testing purposes.

It provides reduced functionality, it supports only - * Servlets.

The servlet container is started in a free port. + * An embedded servlet container for testing purposes. + *

+ * It provides reduced functionality, it supports only Servlets. + *

+ * The servlet container is started in a free port. */ public class EmbeddedServletContainer { private Server server; @@ -36,7 +40,7 @@ public class EmbeddedServletContainer { * Create a servlet container. * * @param contextPath context path for the servlet, it must not be prefixed or append with "/", for the default - * context use "" + * context use "" */ public EmbeddedServletContainer(String contextPath) { this.contextPath = contextPath; @@ -46,11 +50,25 @@ public EmbeddedServletContainer(String contextPath) { server.setHandler(context); } + /** + * Add a filter to the container. + * + * @param path path for the filter, it should be prefixed with '/", it may contain a wild card at the end. + * @param filterClass filter class + */ + public void addFilter(String path, Class filterClass) { + context.addFilter(new FilterHolder(filterClass), "/*", 0); + } + + public void addAttribute(String key, Object object) { + context.setAttribute(key, object); + } + /** * Add a servlet to the container. * * @param servletPath servlet path for the servlet, it should be prefixed with '/", it may contain a wild card at - * the end. + * the end. * @param servletClass servlet class */ public void addServletEndpoint(String servletPath, Class servletClass) { @@ -58,7 +76,9 @@ public void addServletEndpoint(String servletPath, Class servletClass) { } /** - * Start the servlet container.

The container starts on a free port. + * Start the servlet container. + *

+ * The container starts on a free port. * * @throws Exception thrown if the container could not start. */ @@ -136,4 +156,4 @@ public void stop() { port = -1; } -} +} \ No newline at end of file diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index f9d26d496..72c7262dd 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1175,5 +1175,14 @@ library path are used. + + + + authentication.providers + org.apache.hadoop.http.authentication.server.simple.SimpleAuthenticationHeaderProvider + + List of Oozie server authentication providers (separated by commas). + + diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index e6c859f4b..26f02eca2 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -4,16 +4,17 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. See accompanying LICENSE file. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. */ package org.apache.oozie.client; +import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; import java.io.StringReader; @@ -29,7 +30,9 @@ import org.apache.oozie.servlet.JobsServlet; import org.apache.oozie.servlet.MockCoordinatorEngineService; import org.apache.oozie.servlet.MockDagEngineService; +import org.apache.hadoop.http.authentication.web.AuthenticationProcessingFilter; import org.apache.oozie.servlet.V1AdminServlet; +import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; //hardcoding options instead using constants on purpose, to detect changes to option names if any and correct docs. @@ -44,16 +47,30 @@ public class TestOozieCLI extends DagServletTestCase { static final boolean IS_SECURITY_ENABLED = false; static final String VERSION = "/v" + OozieClient.WS_PROTOCOL_VERSION; - static final String[] END_POINTS = {"/versions", VERSION + "/jobs", VERSION + "/job/*", VERSION + "/admin/*"}; - static final Class[] SERVLET_CLASSES = - { HeaderTestingVersionServlet.class, JobsServlet.class, JobServlet.class, + static final String[] END_POINTS = { "/versions", VERSION + "/jobs", VERSION + "/job/*", VERSION + "/admin/*" }; + static final Class[] SERVLET_CLASSES = { HeaderTestingVersionServlet.class, JobsServlet.class, JobServlet.class, V1AdminServlet.class }; + static final String[] FILTER_PATHS = { "/*" }; + static final Class[] FILTER_CLASSES = { AuthenticationProcessingFilter.class }; + + protected void runTestWithAuthFilter(String[] servletPath, Class[] servletClass, boolean securityEnabled, + Callable assertions) throws Exception { + runTest(servletPath, servletClass, FILTER_PATHS, FILTER_CLASSES, securityEnabled, assertions); + } + @Override protected void setUp() throws Exception { super.setUp(); MockDagEngineService.reset(); MockCoordinatorEngineService.reset(); + setSystemProperty("authentication.providers", + "org.apache.hadoop.http.authentication.server.simple.SimpleAuthenticationHeaderProvider"); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); } private String createConfigFile(String appPath) throws Exception { @@ -92,7 +109,7 @@ private String createPropertiesFileWithTrailingSpaces(String appPath) throws Exc props.setProperty(OozieClient.GROUP_NAME, getTestGroup()); injectKerberosInfo(props); props.setProperty(OozieClient.APP_PATH, appPath); - //add spaces to string + // add spaces to string props.setProperty(OozieClient.RERUN_SKIP_NODES + " ", " node "); OutputStream os = new FileOutputStream(path); props.store(os, ""); @@ -101,7 +118,7 @@ private String createPropertiesFileWithTrailingSpaces(String appPath) throws Exc } public void testSubmit() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); int wfCount = MockDagEngineService.INIT_WF_COUNT; @@ -110,32 +127,32 @@ public Void call() throws Exception { getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "workflow.xml")).close(); - String[] args = new String[]{"job", "-submit", "-oozie", oozieUrl, "-config", - createConfigFile(appPath.toString())}; + String[] args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createConfigFile(appPath.toString()) }; assertEquals(0, new OozieCLI().run(args)); assertEquals("submit", MockDagEngineService.did); assertFalse(MockDagEngineService.started.get(wfCount)); wfCount++; - args = new String[]{"job", "-submit", "-oozie", oozieUrl, "-config", - createPropertiesFile(appPath.toString())}; + args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createPropertiesFile(appPath.toString()) }; assertEquals(0, new OozieCLI().run(args)); assertEquals("submit", MockDagEngineService.did); assertFalse(MockDagEngineService.started.get(wfCount)); MockDagEngineService.reset(); wfCount = MockDagEngineService.INIT_WF_COUNT; - args = new String[]{"job", "-submit", "-oozie", oozieUrl, "-config", - createPropertiesFile(appPath.toString()) + "x"}; + args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createPropertiesFile(appPath.toString()) + "x" }; assertEquals(-1, new OozieCLI().run(args)); assertEquals(null, MockDagEngineService.did); try { MockDagEngineService.started.get(wfCount); - //job was not created, then how did this extra job come after reset? fail!! + // job was not created, then how did this extra job come after reset? fail!! fail(); } catch (Exception e) { - //job was not submitted, so its fine + // job was not submitted, so its fine } return null; } @@ -143,7 +160,7 @@ public Void call() throws Exception { } public void testSubmitWithPropertyArguments() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); int wfCount = MockDagEngineService.INIT_WF_COUNT; @@ -152,8 +169,8 @@ public Void call() throws Exception { getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "workflow.xml")).close(); - String[] args = new String[]{"job", "-submit", "-oozie", oozieUrl, "-config", - createConfigFile(appPath.toString()), "-Da=X", "-Db=B"}; + String[] args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createConfigFile(appPath.toString()), "-Da=X", "-Db=B" }; assertEquals(0, new OozieCLI().run(args)); assertEquals("submit", MockDagEngineService.did); assertFalse(MockDagEngineService.started.get(wfCount)); @@ -166,15 +183,15 @@ public Void call() throws Exception { } public void testRun() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "workflow.xml")).close(); String oozieUrl = getContextURL(); int wfCount = MockDagEngineService.INIT_WF_COUNT; - String[] args = new String[]{"job", "-run", "-oozie", oozieUrl, "-config", - createConfigFile(appPath.toString())}; + String[] args = new String[] { "job", "-run", "-oozie", oozieUrl, "-config", + createConfigFile(appPath.toString()) }; assertEquals(0, new OozieCLI().run(args)); assertEquals("submit", MockDagEngineService.did); assertTrue(MockDagEngineService.started.get(wfCount)); @@ -185,16 +202,16 @@ public Void call() throws Exception { } public void testStart() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-start", MockDagEngineService.JOB_ID + "1"}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-start", MockDagEngineService.JOB_ID + "1" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_START, MockDagEngineService.did); assertTrue(MockDagEngineService.started.get(1)); - args = new String[]{"job", "-oozie", oozieUrl, "-start", - MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1)}; + args = new String[] { "job", "-oozie", oozieUrl, "-start", + MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1) }; assertEquals(-1, new OozieCLI().run(args)); return null; } @@ -202,15 +219,15 @@ public Void call() throws Exception { } public void testSuspend() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-suspend", MockDagEngineService.JOB_ID + 1}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-suspend", MockDagEngineService.JOB_ID + 1 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_SUSPEND, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-suspend", - MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1)}; + args = new String[] { "job", "-oozie", oozieUrl, "-suspend", + MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1) }; assertEquals(-1, new OozieCLI().run(args)); return null; } @@ -218,15 +235,15 @@ public Void call() throws Exception { } public void testResume() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-resume", MockDagEngineService.JOB_ID + 1}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-resume", MockDagEngineService.JOB_ID + 1 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_RESUME, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-resume", - MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1)}; + args = new String[] { "job", "-oozie", oozieUrl, "-resume", + MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1) }; assertEquals(-1, new OozieCLI().run(args)); return null; } @@ -234,15 +251,15 @@ public Void call() throws Exception { } public void testKill() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-kill", MockDagEngineService.JOB_ID + 1}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-kill", MockDagEngineService.JOB_ID + 1 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_KILL, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-kill", - MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1)}; + args = new String[] { "job", "-oozie", oozieUrl, "-kill", + MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1) }; assertEquals(-1, new OozieCLI().run(args)); return null; } @@ -250,14 +267,14 @@ public Void call() throws Exception { } public void testReRun() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "workflow.xml")).close(); String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-config", createConfigFile(appPath.toString()), - "-rerun", MockDagEngineService.JOB_ID + "1"}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-config", + createConfigFile(appPath.toString()), "-rerun", MockDagEngineService.JOB_ID + "1" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_RERUN, MockDagEngineService.did); assertTrue(MockDagEngineService.started.get(1)); @@ -272,15 +289,14 @@ public Void call() throws Exception { * @throws Exception */ public void testCoordReRun1() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "coordinator.xml")).close(); String oozieUrl = getContextURL(); String[] args = new String[] { "job", "-oozie", oozieUrl, "-rerun", - MockCoordinatorEngineService.JOB_ID + "1", - "-action", "1" }; + MockCoordinatorEngineService.JOB_ID + "1", "-action", "1" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_COORD_ACTION_RERUN, MockCoordinatorEngineService.did); assertTrue(MockCoordinatorEngineService.started.get(1)); @@ -295,15 +311,14 @@ public Void call() throws Exception { * @throws Exception */ public void testCoordReRun2() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "coordinator.xml")).close(); String oozieUrl = getContextURL(); String[] args = new String[] { "job", "-oozie", oozieUrl, "-rerun", - MockCoordinatorEngineService.JOB_ID + "1", - "-date", "2009-12-15T01:00Z::2009-12-16T01:00Z" }; + MockCoordinatorEngineService.JOB_ID + "1", "-date", "2009-12-15T01:00Z::2009-12-16T01:00Z" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_COORD_ACTION_RERUN, MockCoordinatorEngineService.did); assertTrue(MockCoordinatorEngineService.started.get(1)); @@ -318,15 +333,14 @@ public Void call() throws Exception { * @throws Exception */ public void testCoordReRunNeg1() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "coordinator.xml")).close(); String oozieUrl = getContextURL(); String[] args = new String[] { "job", "-oozie", oozieUrl, "-rerun", - MockCoordinatorEngineService.JOB_ID + "1", - "-date", "2009-12-15T01:00Z", "-action", "1" }; + MockCoordinatorEngineService.JOB_ID + "1", "-date", "2009-12-15T01:00Z", "-action", "1" }; assertEquals(-1, new OozieCLI().run(args)); assertNull(MockCoordinatorEngineService.did); assertFalse(MockCoordinatorEngineService.started.get(1)); @@ -341,7 +355,7 @@ public Void call() throws Exception { * @throws Exception */ public void testCoordReRunNeg2() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath); @@ -358,24 +372,24 @@ public Void call() throws Exception { } public void testJobStatus() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); MockDagEngineService.reset(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 0}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 0 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-localtime", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 1}; + args = new String[] { "job", "-localtime", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 1 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 2}; + args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 2 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-info", - MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1)}; + args = new String[] { "job", "-oozie", oozieUrl, "-info", + MockDagEngineService.JOB_ID + (MockDagEngineService.workflows.size() + 1) }; assertEquals(-1, new OozieCLI().run(args)); return null; } @@ -383,34 +397,34 @@ public Void call() throws Exception { } public void testJobsStatus() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); - String[] args = new String[]{"jobs", "-len", "3", "-offset", "2", "-oozie", oozieUrl, "-filter", - "name=x"}; + String[] args = new String[] { "jobs", "-len", "3", "-offset", "2", "-oozie", oozieUrl, "-filter", + "name=x" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOBS_FILTER_PARAM, MockDagEngineService.did); - args = new String[]{"jobs", "-localtime", "-len", "3", "-offset", "2", "-oozie", oozieUrl, "-filter", - "name=x"}; + args = new String[] { "jobs", "-localtime", "-len", "3", "-offset", "2", "-oozie", oozieUrl, "-filter", + "name=x" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOBS_FILTER_PARAM, MockDagEngineService.did); - args = new String[]{"jobs", "-jobtype", "coord", "-filter", "status=FAILED", "-oozie", oozieUrl}; + args = new String[] { "jobs", "-jobtype", "coord", "-filter", "status=FAILED", "-oozie", oozieUrl }; assertEquals(0, new OozieCLI().run(args)); - assertEquals(RestConstants.JOBS_FILTER_PARAM, MockDagEngineService.did); + assertEquals(RestConstants.JOBS_FILTER_PARAM, MockDagEngineService.did); return null; } }); } public void testHeaderPropagation() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); setSystemProperty(OozieCLI.WS_HEADER_PREFIX + "header", "test"); String oozieUrl = getContextURL(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-start", MockDagEngineService.JOB_ID + 1}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-start", MockDagEngineService.JOB_ID + 1 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_ACTION_START, MockDagEngineService.did); assertTrue(HeaderTestingVersionServlet.OOZIE_HEADERS.containsKey("header")); @@ -422,15 +436,15 @@ public Void call() throws Exception { } public void testOozieStatus() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); String oozieUrl = getContextURL(); - String[] args = new String[]{"admin", "-status", "-oozie", oozieUrl}; + String[] args = new String[] { "admin", "-status", "-oozie", oozieUrl }; assertEquals(0, new OozieCLI().run(args)); - args = new String[]{"admin", "-oozie", oozieUrl, "-systemmode", "NORMAL"}; + args = new String[] { "admin", "-oozie", oozieUrl, "-systemmode", "NORMAL" }; assertEquals(0, new OozieCLI().run(args)); return null; } @@ -438,12 +452,12 @@ public Void call() throws Exception { } public void testServerBuildVersion() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); String oozieUrl = getContextURL(); - String[] args = new String[]{"admin", "-version", "-oozie", oozieUrl}; + String[] args = new String[] { "admin", "-version", "-oozie", oozieUrl }; assertEquals(0, new OozieCLI().run(args)); return null; @@ -452,28 +466,30 @@ public Void call() throws Exception { } public void testClientBuildVersion() throws Exception { - String[] args = new String[]{"version"}; + String[] args = new String[] { "version" }; assertEquals(0, new OozieCLI().run(args)); } public void testJobInfo() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); MockDagEngineService.reset(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 0}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 0 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 1, "-len", "3", "-offset", "1"}; + args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 1, "-len", "3", + "-offset", "1" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 2, "-len", "2"}; + args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 2, "-len", "2" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); - args = new String[]{"job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 3, "-offset", "3"}; + args = new String[] { "job", "-oozie", oozieUrl, "-info", MockDagEngineService.JOB_ID + 3, "-offset", + "3" }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); @@ -483,37 +499,36 @@ public Void call() throws Exception { } public void testJobLog() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); MockDagEngineService.reset(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-log", MockDagEngineService.JOB_ID + 0}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-log", MockDagEngineService.JOB_ID + 0 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_LOG, MockDagEngineService.did); - return null; } }); } public void testJobDefinition() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { String oozieUrl = getContextURL(); MockDagEngineService.reset(); - String[] args = new String[]{"job", "-oozie", oozieUrl, "-definition", MockDagEngineService.JOB_ID + 0}; + String[] args = new String[] { "job", "-oozie", oozieUrl, "-definition", + MockDagEngineService.JOB_ID + 0 }; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_DEFINITION, MockDagEngineService.did); - return null; } }); } public void testPropertiesWithTrailingSpaces() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { MockDagEngineService.reset(); String oozieUrl = getContextURL(); @@ -522,8 +537,8 @@ public Void call() throws Exception { getFileSystem().mkdirs(appPath); getFileSystem().create(new Path(appPath, "workflow.xml")).close(); - String[] args = new String[]{"job", "-submit", "-oozie", oozieUrl, "-config", - createPropertiesFileWithTrailingSpaces(appPath.toString())}; + String[] args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createPropertiesFileWithTrailingSpaces(appPath.toString()) }; assertEquals(0, new OozieCLI().run(args)); assertEquals("submit", MockDagEngineService.did); String confStr = MockDagEngineService.workflows.get(MockDagEngineService.INIT_WF_COUNT).getConf(); @@ -536,16 +551,79 @@ public Void call() throws Exception { } public void testAdminQueueDump() throws Exception { - runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); String oozieUrl = getContextURL(); - String[] args = new String[]{"admin", "-queuedump", "-oozie", oozieUrl}; + String[] args = new String[] { "admin", "-queuedump", "-oozie", oozieUrl }; assertEquals(0, new OozieCLI().run(args)); return null; } }); } + + private String prepareClientSite(String clientSite) throws Exception { + File siteFile = new File(getTestCaseConfDir(), clientSite); + IOUtils.copyStream(IOUtils.getResourceAsStream(clientSite, -1), new FileOutputStream(siteFile)); + return siteFile.getAbsolutePath(); + } + + public void testAdminWithAuth() throws Exception { + final String siteFile = prepareClientSite("client-site-test.xml"); + + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + + String oozieUrl = getContextURL(); + String[] args = new String[] { "admin", "-version", "-oozie", oozieUrl, "-auth", "simple", + "-clientsite", siteFile }; + assertEquals(0, new OozieCLI().run(args)); + + return null; + } + }); + } + + public void testSubmitWithAuth() throws Exception { + final String siteFile = prepareClientSite("client-site-test.xml"); + + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + String oozieUrl = getContextURL(); + int wfCount = MockDagEngineService.INIT_WF_COUNT; + + Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + getFileSystem().create(new Path(appPath, "workflow.xml")).close(); + + String[] args = new String[] { "job", "-submit", "-oozie", oozieUrl, "-config", + createConfigFile(appPath.toString()), "-auth", "simple", "-clientsite", siteFile }; + assertEquals(0, new OozieCLI().run(args)); + assertEquals("submit", MockDagEngineService.did); + assertFalse(MockDagEngineService.started.get(wfCount)); + + return null; + } + }); + } + + public void testJobsWithAuth() throws Exception { + final String siteFile = prepareClientSite("client-site-test.xml"); + + runTestWithAuthFilter(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + String oozieUrl = getContextURL(); + String[] args = new String[] { "jobs", "-len", "3", "-offset", "2", "-oozie", oozieUrl, "-filter", + "name=x", "-auth", "simple", "-clientsite", siteFile }; + assertEquals(0, new OozieCLI().run(args)); + assertEquals(RestConstants.JOBS_FILTER_PARAM, MockDagEngineService.did); + + return null; + } + }); + } + } diff --git a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java index 94e3c89b6..0d868cd92 100644 --- a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java +++ b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java @@ -14,6 +14,11 @@ */ package org.apache.oozie.servlet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.authentication.web.AuthenticationProcessingFilter; +import org.apache.hadoop.http.authentication.web.CookieSignerVerifier; +import org.apache.hadoop.http.authentication.web.FileSystemEvictorCallback; +import org.apache.hadoop.http.authentication.web.ProxyUGICacheManager; import org.apache.oozie.service.AuthorizationService; import org.apache.oozie.service.Services; @@ -24,8 +29,10 @@ import java.net.URL; import java.net.URLEncoder; +import java.util.Date; import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; public abstract class DagServletTestCase extends XFsTestCase { private EmbeddedServletContainer container; @@ -44,8 +51,8 @@ protected URL createURL(String servletPath, String resource, Map if (parameters.size() > 0) { String separator = "?"; for (Map.Entry param : parameters.entrySet()) { - sb.append(separator).append(URLEncoder.encode(param.getKey(), "UTF-8")).append("=") - .append(URLEncoder.encode(param.getValue(), "UTF-8")); + sb.append(separator).append(URLEncoder.encode(param.getKey(), "UTF-8")).append("=").append( + URLEncoder.encode(param.getValue(), "UTF-8")); separator = "&"; } } @@ -59,11 +66,16 @@ protected URL createURL(String resource, Map parameters) throws @SuppressWarnings("unchecked") protected void runTest(String servletPath, Class servletClass, boolean securityEnabled, Callable assertions) throws Exception { - runTest(new String[]{servletPath}, new Class[]{servletClass}, securityEnabled, assertions); + runTest(new String[] { servletPath }, new Class[] { servletClass }, securityEnabled, assertions); } protected void runTest(String[] servletPath, Class[] servletClass, boolean securityEnabled, - Callable assertions) throws Exception { + Callable assertions) throws Exception { + runTest(servletPath, servletClass, new String[0], new Class[0], securityEnabled, assertions); + } + + protected void runTest(String[] servletPath, Class[] servletClass, String[] filterPath, Class[] filterClass, + boolean securityEnabled, Callable assertions) throws Exception { Services services = new Services(); this.servletPath = servletPath[0]; try { @@ -77,6 +89,17 @@ protected void runTest(String[] servletPath, Class[] servletClass, boolean secur for (int i = 0; i < servletPath.length; i++) { container.addServletEndpoint(servletPath[i], servletClass[i]); } + for (int i = 0; i < filterPath.length; i++) { + container.addFilter(filterPath[i], filterClass[i]); + } + // ***** START setup filter properties ***** + container.addAttribute(AuthenticationProcessingFilter.AUTH_CONFIGURATION, services.getConf()); + container.addAttribute("StartTime", new Date()); + container.addAttribute(AuthenticationProcessingFilter.COOKIE_SIGNER_VERIFIER, + new CookieSignerVerifier(services.getConf())); + container.addAttribute(AuthenticationProcessingFilter.UGI_CACHE_MANAGER, + initializeUGICacheManager(services.getConf())); + // ***** END setup filter properties ***** container.start(); assertions.call(); } @@ -90,4 +113,13 @@ protected void runTest(String[] servletPath, Class[] servletClass, boolean secur } } + protected ProxyUGICacheManager initializeUGICacheManager(Configuration conf) { + long ugiExpiryTimeInMillis = conf.getLong("ugi.expirytime.in.millis", TimeUnit.MINUTES.toMillis(10)); + long evictionIntervalInMillis = conf.getLong("ugi.evictioninterval.in.millis", TimeUnit.MINUTES.toMillis(5)); + + ProxyUGICacheManager cacheManager = new ProxyUGICacheManager(ugiExpiryTimeInMillis, evictionIntervalInMillis, + new FileSystemEvictorCallback()); + return cacheManager; + } + } diff --git a/core/src/test/resources/client-site-test.xml b/core/src/test/resources/client-site-test.xml new file mode 100644 index 000000000..3afd61940 --- /dev/null +++ b/core/src/test/resources/client-site-test.xml @@ -0,0 +1,29 @@ + + + + + + + + oozie.auth.map + simple=org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator + + List of Oozie client authenticator (separated by commas). + The format is key=classname and key will be used in command line for -auth options. + + + + \ No newline at end of file diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki index 8f984b217..5edd9770f 100644 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@ -48,19 +48,19 @@ if necessary use =sudo -u OOZIE_USER= when invoking the scripts. Use the =oozie-setup.sh= script to add the Hadoop JARs and the ExtJS library to Oozie. -$ oozie bin/oozie-setup.sh -hadoop 0.20.200 ${HADOOP_HOME} -extjs /tmp/ext-2.2.zip +$ bin/oozie-setup.sh -hadoop 0.20.200 ${HADOOP_HOME} -extjs /tmp/ext-2.2.zip To start Oozie as a daemon process run: -$ oozie bin/oozie-start.sh +$ bin/oozie-start.sh To start Oozie as a foreground process run: -$ oozie bin/oozie-run.sh +$ bin/oozie-run.sh Check the Oozie log file =logs/oozie.log= to ensure Oozie started properly. diff --git a/docs/src/site/twiki/OozieAuthSpec.twiki b/docs/src/site/twiki/OozieAuthSpec.twiki new file mode 100644 index 000000000..edb661e41 --- /dev/null +++ b/docs/src/site/twiki/OozieAuthSpec.twiki @@ -0,0 +1,120 @@ + + +[[index][::Go back to Oozie Documentation Index::]] + +----- + +---+!! Oozie Authentication Specification + +The goal of this document is to provide developer a tutorial of how to write your own authentication and configure in the run-time environment of a Oozie server. + +%TOC% + + +---++ 1 Oozie Authentication Definitions + +*Authenticator:* A client side class to authenticate user and send the authentication information to server along with each request. + +*AuthenticationProvider:* A server side component to retrieve authentication token from http request and validate the token. + +*AuthenticationToken:* A object contains authentication information for a request. + +---++ 2 Oozie Authentication Introduction + +Oozie Authentication provides a framework to let developer provide a custom implementation to authenticate the requests from +Oozie client. The client side authentication code is used to send the authentication information as a header in the HTTP request +sent to the server. It can be used to send different kinds of authentication tokens or certificates. After a successful +authentication using one of the configured methods, it sends Hadoop-HTTP-Auth cookie in further requests. + +The server side authentication module has a AuthenticationProviderFactory which needs to be initialized with the required +AuthenticationProviders from a configuration file. The authentication is handled by the AuthenticationProcessingFilter. +Once a request is received the following happens in the filter: + + * Request is checked for presence of Hadoop-HTTP-Auth cookie. If present the signature is verified and + the username is extracted from the cookie. + * If the cookie is not present or is invalid, a supported provider is fetched from the AuthenticationProviderFactory + passing in the Request. If there is no supported provider a 401 is sent with the header "WWW-Authenticate: Negotiate" + * If a supported provider is found, the authentication is delegated to the supported provider. On successful authentication + the provider returns an instance of AuthenticationToken which contains the authenticated user. + * An UGI object is constructed out of the authenticated user and set as the request attribute "authorized.ugi" which + can be later consumed by the servlets. + +---++ 3 Server Authentication Implementation + +To write a new custom authentication for server, two classes have to be provided with overrided implementation. + +*Provider:* three methods are required to implement. + * supports() : the method checks if its authentication mechanism supports the authentication information a request provided. + * getAuthenticationToken() : the method is called after supports() returns true and used to constructs the token from parameters in a request. + * authenticate() : the method is used to validate a token created above and rewrite it with new information if needed. + +*Token:* the instance contains the information for a authentication provider to use. + +For example, + +SimpleAuthenticationHeadProvider implements the AuthenticationProvider to check if client has send a parameter (username) in the request. + +SimpleAuthenticationToken extends AbstractAuthenticationToken to set the authentication flag to true and save client parameter in a instance of Token. + +---++ 4 Client Authentication Implementation + +To write a new custom authentication for client, one class have to be provided with overrided implementation. + +*HttpAuthenticator:* one methods is required to implement. + * authenticate(conf, connection) : the method is used to do the client authentication and insert authentication information to http connection. + +For example, + +SimpleAuthenticator extends the HttpAuthenticator and send user name as http connection's request parameter. + +---++ 5 Server Authentication Configuration + +An authentication provider can be given in "oozie-site.xml" for Oozie server. The property "authentication.providers" is used to configure +what authentication mechanisms are supported in Oozie server runtime. + + + authentication.providers + org.apache.hadoop.http.authentication.server.simple.SimpleAuthenticationHeaderProvider + Comma separated list of authentication providers in FQCN. + + +---++ 6 Client Authentication Configuration + +To use an authenticator in client, a configuration "oozie.auth.map" has to be defined in client-site.xml or other user-specified client site xml. The value of "oozie.auth.map" specifies the mapping from key “XXX” to “org.apache.XXXauthenticator”. The format of "oozie.auth.map" is like "key1=value1,key2=value2". A key “XXX” is used at command line to specify which authenticator to use. + +For example, + +Default client-site.xml in Oozie Client can be found in oozie-client tar. + + + + + + oozie.auth.map + simple=org.apache.hadoop.http.authentication.client.simple.SimpleAuthenticator + + List of Oozie client authenticator (separated by commas). + The format is key=classname and key will be used in command line for -auth options. + + + + + +* Command line option "-clientsite" or environment variable "CLIENT_SITE" is used to specify the client site xml. + +---++ 7 Client Authentication CommandLine + +In command line, argument "-auth" has to be given to use user-defined authenticator. Default is SimpleAuthenticator if "-auth" is not given. A command line sample is: + + Ex. oozie job –run –config map-red.properties –auth XXX -clientsite PATH_TO_CLIENT_SITE -Dkey1=value –Dkey2=value + +The value of "-clientsite" or environment variable "CLIENT_SITE" is the path to the client site xml which contains client site configuration. The value of "-auth" is a key in the value "oozie.auth.map" of client site xml. When "-auth" is used, Oozie client looks for "-clientsite", or env variable "CLIENT_SITE" if "-clientsite" is not present. + +For example, + +To use simple authenticator from above example, + +$ oozie job –run –config map-red.properties –auth simple -clientsite PATH_TO_CLIENT_SITE + + + diff --git a/docs/src/site/twiki/index.twiki b/docs/src/site/twiki/index.twiki index 247d14f36..48b5b4954 100644 --- a/docs/src/site/twiki/index.twiki +++ b/docs/src/site/twiki/index.twiki @@ -44,6 +44,7 @@ Enough reading already? Follow the steps in [[DG_QuickStart][Oozie Quick Start]] * [[DG_UsingHadoopKerberos][Using a Hadoop cluster with Kerberos Authentication]] * [[DG_CustomActionExecutor][Writing a Custom Action Executor]] + * [[OozieAuthSpec][Writing a Custom Authentication]] * [[./apidocs/index.html][Oozie Javadocs]] ---++ Administrator Documentation diff --git a/pom.xml b/pom.xml index 5b2d4a91b..5f72da9ab 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ + authentication client core webapp @@ -121,7 +122,11 @@ - + + com.yahoo.oozie + oozie-authentication + 2.3.0-SNAPSHOT + com.yahoo.oozie oozie-client @@ -220,7 +225,7 @@ commons-codec commons-codec - 1.3 + 1.4 @@ -233,6 +238,10 @@ org.apache.commons commons-cli + + org.slf4j + slf4j-log4j12 + diff --git a/src/main/assemblies/client.xml b/src/main/assemblies/client.xml index 75aad515d..ae8d98570 100644 --- a/src/main/assemblies/client.xml +++ b/src/main/assemblies/client.xml @@ -44,6 +44,14 @@ 0755 + + + ${basedir}/src/main/conf + oozie-client-${project.version}/conf + + * + + diff --git a/src/main/assemblies/distro.xml b/src/main/assemblies/distro.xml index 35bfdc369..ee23eaf14 100644 --- a/src/main/assemblies/distro.xml +++ b/src/main/assemblies/distro.xml @@ -64,6 +64,13 @@ * + + ${basedir}/../client/target/oozie-client-${project.version}-client/oozie-client-${project.version}/conf + /conf + + * + + ${basedir}/target/tomcat/oozie-server diff --git a/webapp/pom.xml b/webapp/pom.xml index 1ae21fc26..427495ee3 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -67,14 +67,6 @@ javax.servlet jsp-api - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - commons-logging commons-logging-api diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index 542960b4c..006d414e9 100644 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -17,6 +17,97 @@ OOZIE + + + AuthenticationProcessingFilter + org.apache.hadoop.http.authentication.web.AuthenticationProcessingFilter + + + + OozieAuthFilter + org.apache.oozie.filter.OozieAuthFilter + + + + + + AuthenticationProcessingFilter + versions + + + + AuthenticationProcessingFilter + v0admin + + + AuthenticationProcessingFilter + v1admin + + + + AuthenticationProcessingFilter + v0jobs + + + AuthenticationProcessingFilter + v1jobs + + + + AuthenticationProcessingFilter + v0job + + + + AuthenticationProcessingFilter + v1job + + + + AuthenticationProcessingFilter + sla-event + + + + + + OozieAuthFilter + versions + + + + OozieAuthFilter + v0admin + + + OozieAuthFilter + v1admin + + + + OozieAuthFilter + v0jobs + + + OozieAuthFilter + v1jobs + + + + OozieAuthFilter + v0job + + + + OozieAuthFilter + v1job + + + + OozieAuthFilter + sla-event + +