[WIP] Migrate to jetty 12 with EE8 and bump java servlet to 4.0.1

hbase-http/pom.xml

@@ -78,7 +78,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hbase.thirdparty</groupId>
-      <artifactId>hbase-shaded-jetty</artifactId>
+      <artifactId>hbase-shaded-jetty-ee8</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hbase.thirdparty</groupId>

hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java

@@ -24,7 +24,7 @@
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet;
 
 /**
  * General servlet which is admin-authorized.

hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java

@@ -72,6 +72,12 @@
 import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterMapping;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration;
@@ -84,18 +90,9 @@
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ErrorHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.HandlerCollection;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.RequestLogHandler;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterMapping;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.util.MultiException;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext;
 import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig;
 import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer;
 
@@ -609,23 +606,21 @@ private void initializeWebServer(String name, String hostName, Configuration con
 
     Preconditions.checkNotNull(webAppContext);
 
-    HandlerCollection handlerCollection = new HandlerCollection();
+    Handler.Sequence handlers = new Handler.Sequence();
 
     ContextHandlerCollection contexts = new ContextHandlerCollection();
     RequestLog requestLog = HttpRequestLog.getRequestLog(name);
 
     if (requestLog != null) {
-      RequestLogHandler requestLogHandler = new RequestLogHandler();
-      requestLogHandler.setRequestLog(requestLog);
-      handlerCollection.addHandler(requestLogHandler);
+      webServer.setRequestLog(requestLog);
     }
 
     final String appDir = getWebAppsPath(name);
 
-    handlerCollection.addHandler(contexts);
-    handlerCollection.addHandler(webAppContext);
+    handlers.addHandler(contexts);
+    handlers.addHandler(webAppContext);
 
-    webServer.setHandler(handlerCollection);
+    webServer.setHandler(handlers);
 
     webAppContext.setAttribute(ADMINS_ACL, adminsAcl);
 
@@ -670,8 +665,9 @@ private void initializeWebServer(String name, String hostName, Configuration con
     // Check if disable stack trace property is configured
     if (!conf.getBoolean(HTTP_UI_SHOW_STACKTRACE_KEY, true)) {
       // Disable stack traces for server errors in UI
-      webServer.setErrorHandler(new ErrorHandler());
-      webServer.getErrorHandler().setShowStacks(false);
+      ErrorHandler errorHanlder = new ErrorHandler();
+      errorHanlder.setShowStacks(false);
+      webServer.setErrorHandler(errorHanlder);
       // Disable stack traces for web app errors in UI
       webAppContext.getErrorHandler().setShowStacks(false);
     }
@@ -796,7 +792,7 @@ private void configureAliasChecks(ServletContextHandler context, boolean shouldS
       if (context.getAliasChecks().stream().anyMatch(aliasCheckerClass::isInstance)) {
         LOG.debug("{} is already part of alias check list", aliasCheckerClass.getName());
       } else {
-        context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context));
+        context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context.getCoreContextHandler()));
         LOG.debug("{} added to the alias check list", aliasCheckerClass.getName());
       }
       LOG.info("Serving aliases allowed for /logs context");
@@ -1213,14 +1209,14 @@ public void start() throws IOException {
       } catch (IOException ex) {
         LOG.info("HttpServer.start() threw a non Bind IOException", ex);
         throw ex;
-      } catch (MultiException ex) {
+      } catch (Exception ex) {
         LOG.info("HttpServer.start() threw a MultiException", ex);
         throw ex;
       }
       // Make sure there is no handler failures.
-      Handler[] handlers = webServer.getHandlers();
-      for (int i = 0; i < handlers.length; i++) {
-        if (handlers[i].isFailed()) {
+      List<Handler> handlers = webServer.getHandlers();
+      for (Handler handler : handlers) {
+        if (handler.isFailed()) {
           throw new IOException("Problem in starting http server. Server handlers failed");
         }
       }
@@ -1290,7 +1286,7 @@ void openListeners() throws Exception {
    * stop the server
    */
   public void stop() throws Exception {
-    MultiException exception = null;
+    Exception exception = null;
     for (ListenerInfo li : listeners) {
       if (!li.isManaged) {
         continue;
@@ -1300,7 +1296,7 @@ public void stop() throws Exception {
         li.listener.close();
       } catch (Exception e) {
         LOG.error("Error while stopping listener for webapp" + webAppContext.getDisplayName(), e);
-        exception = addMultiException(exception, e);
+        exception.addSuppressed(e);
       }
     }
 
@@ -1311,28 +1307,41 @@ public void stop() throws Exception {
     } catch (Exception e) {
       LOG.error("Error while stopping web app context for webapp " + webAppContext.getDisplayName(),
         e);
-      exception = addMultiException(exception, e);
+      exception.addSuppressed(e);
     }
 
     try {
       webServer.stop();
     } catch (Exception e) {
       LOG.error("Error while stopping web server for webapp " + webAppContext.getDisplayName(), e);
-      exception = addMultiException(exception, e);
+      exception.addSuppressed(e);
     }
 
     if (exception != null) {
-      exception.ifExceptionThrow();
+      ifExceptionThrow(exception);
     }
 
   }
 
-  private MultiException addMultiException(MultiException exception, Exception e) {
-    if (exception == null) {
-      exception = new MultiException();
+  /**
+   * Throw a {@link Throwable} as a checked {@link Exception} if it cannot be thrown as unchecked.
+   * @param throwable The {@link Throwable} to throw or null.
+   * @throws Error     If the passed {@link Throwable} is an {@link Error}.
+   * @throws Exception Otherwise, if the passed {@link Throwable} is not null.
+   *
+   * NOTE: This method is copied from Jetty-9
+   */
+  public static void ifExceptionThrow(Throwable throwable) throws Error, Exception {
+    if (throwable == null) {
+      return;
+    }
+    if (throwable instanceof Error error) {
+      throw error;
+    }
+    if (throwable instanceof Exception exception) {
+      throw exception;
     }
-    exception.add(e);
-    return exception;
+    throw new RuntimeException(throwable);
   }
 
   public void join() throws InterruptedException {

hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java

@@ -22,11 +22,11 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.yetus.audience.InterfaceAudience;
 
-import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintMapping;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintSecurityHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.util.security.Constraint;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.ServletConstraint;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintMapping;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintSecurityHandler;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler;
 
 /**
  * HttpServer utility.
@@ -43,7 +43,7 @@ public final class HttpServerUtil {
    */
   public static void constrainHttpMethods(ServletContextHandler ctxHandler,
     boolean allowOptionsMethod) {
-    Constraint c = new Constraint();
+    ServletConstraint c = new ServletConstraint();
     c.setAuthenticate(true);
 
     ConstraintMapping cmt = new ConstraintMapping();

hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java

@@ -29,7 +29,7 @@
 import org.apache.yetus.audience.InterfaceAudience;
 
 import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder;
 
 /**
  * Create a Jetty embedded server to answer http requests. The primary goal is to serve up status

hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java

@@ -27,7 +27,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet;
 
 /**
  * Servlet to serve files generated by {@link ProfileServlet}

hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java

@@ -507,7 +507,7 @@ public void testRequestQuoterWithNotNull() {
 
   @SuppressWarnings("unchecked")
   private static Map<String, Object> parse(String jsonString) {
-    return (Map<String, Object>) JSON.parse(jsonString);
+    return (Map<String, Object>) new JSON().fromJSON(jsonString);
   }
 
   @Test

hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java

@@ -73,7 +73,7 @@ public void testWriteJson() throws Exception {
     Set<String> programSet = new HashSet<>();
     programSet.add("programatically");
     programSet.add("programmatically");
-    Object parsed = JSON.parse(json);
+    Object parsed = new JSON().fromJSON(json);
     Object[] properties = ((Map<String, Object[]>) parsed).get("properties");
     for (Object o : properties) {
       Map<String, Object> propertyInfo = (Map<String, Object>) o;

hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java

@@ -54,7 +54,7 @@ public Response get(@PathParam(PATH) @DefaultValue("UNKNOWN_" + PATH) final Stri
     final Map<String, Object> m = new TreeMap<>();
     m.put(PATH, path);
     m.put(OP, op);
-    final String js = JSON.toString(m);
+    final String js = new JSON().toJSON(m);
     return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
   }
 }

hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java

@@ -28,17 +28,21 @@
 import java.util.regex.Pattern;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.AbstractHandler;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Response;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.util.Callback;
 import org.junit.rules.ExternalResource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.CustomRequestLog;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.Request;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLog;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.AbstractHandler;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.util.RegexSet;
 
 /**
@@ -115,13 +119,12 @@ protected void after() {
   }
 
   private static RequestLog buildRequestLog() {
-    final Slf4jRequestLog requestLog = new Slf4jRequestLog();
-    requestLog.setLoggerName(LOG.getName() + ".RequestLog");
-    requestLog.setExtended(true);
-    return requestLog;
+    Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter();
+    writer.setLoggerName(LOG.getName() + ".RequestLog");
+    return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT);
   }
 
-  private static class MockHandler extends AbstractHandler {
+  private static class MockHandler extends Handler.Abstract {
 
     private final ReadWriteLock responseMappingLock = new ReentrantReadWriteLock();
     private final Map<String, BiConsumer<String, HttpServletResponse>> responseMapping =
@@ -151,6 +154,32 @@ void clearRegistrations() {
     }
 
     @Override
+    public boolean handle(org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request request, Response response, Callback callback) throws Exception {
+      // TODO: TestRESTApiClusterManager is failing because of this line??
+      String target = request.getHttpURI().asString();
+      responseMappingLock.readLock().lock();
+      try {
+        if (!regexSet.matches(target)) {
+          response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+          callback.succeeded();
+          return true;
+        }
+        responseMapping.entrySet().stream()
+          .filter(e -> Pattern.matches(e.getKey(), target))
+          .findAny()
+          .map(Map.Entry::getValue)
+          .orElseThrow(() -> noMatchFound(target))
+          .accept(target, (HttpServletResponse) response);
+        callback.succeeded();
+      } catch (Exception e) {
+        callback.failed(e);
+      } finally {
+        responseMappingLock.readLock().unlock();
+      }
+      return true;
+    }
+
+    //@Override
     public void handle(final String target, final Request baseRequest,
       final HttpServletRequest request, final HttpServletResponse response) {
       responseMappingLock.readLock().lock();
@@ -168,8 +197,7 @@ public void handle(final String target, final Request baseRequest,
     }
 
     private static RuntimeException noMatchFound(final String target) {
-      return new RuntimeException(
-        String.format("Target path '%s' matches no registered regex.", target));
+      return new RuntimeException(String.format("Target path '%s' matches no registered regex.", target));
     }
   }
 }

hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java

@@ -71,7 +71,7 @@ public Response get(final @Context ServletContext context, final @Context UriInf
       return Response.ok(rowModel).build();
     } catch (IOException e) {
       servlet.getMetrics().incrementFailedGetRequests(1);
-      throw new RuntimeException("Cannot retrieve list of namespaces.");
+      throw new RuntimeException("Cannot retrieve list of namespaces.", e);
     }
   }
 

hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java

@@ -57,6 +57,9 @@
 import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
 import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
 import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.jmx.MBeanContainer;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration;
@@ -65,9 +68,6 @@
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -286,10 +286,10 @@
 import org.apache.hbase.thirdparty.com.google.gson.JsonParseException;
 import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors;
 import org.apache.hbase.thirdparty.com.google.protobuf.Service;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
 import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder;
-import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext;
 import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig;
 import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer;
 

hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -96,6 +96,8 @@ allowed_expr+="|^PropertyList-1.0.dtd$"
 # Shaded jetty resources
 allowed_expr+="|^about.html$"
 allowed_expr+="|^jetty-dir.css$"
+# Required by jetty 12 on ee8
+allowed_expr="(|^javax/$)"
 
 
 if [ -n "${allow_hadoop}" ]; then