YARN-11937. Yarn Proxy Behind a Reverse Proxy

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -3026,6 +3026,8 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String PROXY_BIND_HOST =
       PROXY_PREFIX + "bind-host";
 
+  public static final String PROXY_REDIRECT_FLAG = PROXY_PREFIX + "redirect-flag";
+
   /**
    * YARN Service Level Authorization
    */

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -2832,6 +2832,15 @@
     <value>60000</value>
   </property>
 
+  <property>
+    <description>Optional query parameter name that signals the YARN WebAppProxy to redirect
+      the user to the application's tracking URL instead of proxying the request.
+      When the tracking URL contains this flag with the value "true", the proxy
+      performs an HTTP redirect (302) to the tracking URL.</description>
+    <name>yarn.web-proxy.redirect-flag</name>
+    <value/>
+  </property>
+
   <!-- Applications' Configuration -->
 
   <property>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java

@@ -529,6 +529,35 @@ private void methodAction(final HttpServletRequest req,
         default:
           // fall out of the switch
       }
+
+      /*
+       * If the application registered its tracking URL with the configured
+       * redirect flag, the proxy should not attempt
+       * to fetch the resource itself. Instead, it performs an HTTP redirect
+       * to the tracking URL.
+       *
+       * This is required for deployments where the tracking URL is served
+       * behind an external reverse proxy (for example Apache Knox) that is
+       * responsible for routing requests to multiple backend services
+       * such as Spark History Server instances in an HA setup.
+       *
+       * In such environments the YARN WebAppProxy cannot correctly proxy the
+       * request because the reverse proxy expects the request to originate
+       * directly from the user's browser and may require authentication
+       * context (e.g. a JWT) that the YARN proxy must not forward for
+       * security reasons.
+       *
+       * By redirecting the user instead of proxying the request, the browser
+       * sends a new request to the external reverse proxy which can then
+       * handle authentication and route the request to the appropriate
+       * backend service.
+       */
+      String redirectFlagName = conf.get(YarnConfiguration.PROXY_REDIRECT_FLAG, "");
+      if (!redirectFlagName.isBlank() && toFetch.getQuery().equals(redirectFlagName + "=true")) {
+        ProxyUtils.sendRedirect(req, resp, toFetch.toString());
+        return;
+      }
+
       Cookie c = null;
       if (userWasWarned && userApproved) {
         c = makeCheckCookie(id, true);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java

@@ -38,6 +38,7 @@
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -51,6 +52,7 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -333,6 +335,47 @@ void testWebAppProxyConnectionTimeout()
 
   }
 
+  @Test
+  void testRedirectFlagProxyServlet() throws IOException, ServletException {
+    HttpServletRequest request = mock(HttpServletRequest.class);
+    when(request.getMethod()).thenReturn("GET");
+    when(request.getRemoteUser()).thenReturn("dr.who");
+    when(request.getPathInfo()).thenReturn("/application_00_0");
+    when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration());
+
+    HttpServletResponse response = mock(HttpServletResponse.class);
+    when(response.getOutputStream()).thenReturn(mock(ServletOutputStream.class));
+    when(response.getWriter()).thenReturn(mock(PrintWriter.class));
+    WebAppProxyServlet servlet = new WebAppProxyServlet();
+    ServletConfig config = mock(ServletConfig.class);
+    ServletContext context = mock(ServletContext.class);
+    when(config.getServletContext()).thenReturn(context);
+    AppReportFetcherForTest appReportFetcher =
+        new AppReportFetcherForTest(new YarnConfiguration());
+    servlet.init(config);
+    when(config.getServletContext()
+        .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE))
+        .thenReturn(appReportFetcher);
+
+    appReportFetcher.answer = 8;
+
+    //Check if flag is on
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.set(YarnConfiguration.PROXY_REDIRECT_FLAG, "yarn_knox_proxy");
+    servlet.setConf(conf);
+    servlet.doGet(request, response);
+
+    //Check if flag is off
+    conf.set(YarnConfiguration.PROXY_REDIRECT_FLAG, "");
+    servlet.setConf(conf);
+    servlet.doGet(request, response);
+
+    ArgumentCaptor<Integer> statusCaptor = ArgumentCaptor.forClass(Integer.class);
+    Mockito.verify(response, Mockito.times(2)).setStatus(statusCaptor.capture());
+    assertEquals(HttpServletResponse.SC_FOUND, statusCaptor.getAllValues().get(0));
+    assertEquals(HttpServletResponse.SC_OK, statusCaptor.getAllValues().get(1));
+  }
+
   @Test
   @Timeout(5000)
   void testAppReportForEmptyTrackingUrl() throws Exception {
@@ -654,6 +697,11 @@ public FetchedAppReport getApplicationReport(ApplicationId appId)
         result.getApplicationReport().setOriginalTrackingUrl("localhost:"
             + originalPort + "/foo/timeout?a=b#main");
         return result;
+      } else if (answer == 8) {
+        FetchedAppReport result = getDefaultApplicationReport(appId);
+        result.getApplicationReport().setOriginalTrackingUrl("localhost:"
+            + originalPort + "/foo/bar?yarn_knox_proxy=true");
+        return result;
       }
       return null;
     }