Merge branch 'apache:trunk' into YARN-11262-PART4

Author: slfan1989

BUILDING.txt

@@ -6,7 +6,7 @@ Requirements:
 * Unix System
 * JDK 1.8
 * Maven 3.3 or later
-* Boost 1.72 (if compiling native code)
+* Boost 1.86.0 (if compiling native code)
 * Protocol Buffers 3.21.12 (if compiling native code)
 * CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
@@ -82,8 +82,8 @@ Refer to  dev-support/docker/Dockerfile):
   $ make -j$(nproc)
   $ sudo make install
 * Boost
-  $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
-  $ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
+  $ curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download > boost_1_86_0.tar.bz2
+  $ tar --bzip2 -xf boost_1_86_0.tar.bz2 && cd boost_1_86_0
   $ ./bootstrap.sh --prefix=/usr/
   $ ./b2 --without-python
   $ sudo ./b2 --without-python install
@@ -493,9 +493,9 @@ Building on CentOS 8
   $ sudo make install
 
 * Install boost.
-  $ curl -L -o boost_1_72_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download
-  $ tar xjf boost_1_72_0.tar.bz2
-  $ cd boost_1_72_0
+  $ curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download
+  $ tar xjf boost_1_86_0.tar.bz2
+  $ cd boost_1_86_0
   $ ./bootstrap.sh --prefix=/usr/local
   $ ./b2
   $ sudo ./b2 install
@@ -526,7 +526,7 @@ Requirements:
 * Windows 10
 * JDK 1.8
 * Maven 3.0 or later (maven.apache.org)
-* Boost 1.72 (boost.org)
+* Boost 1.86.0 (boost.org)
 * Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)
 * Visual Studio 2019 (visualstudio.com)

dev-support/docker/pkg-resolver/install-boost.sh

@@ -27,25 +27,25 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-default_version="1.72.0"
+default_version="1.86.0"
 version_to_install=$default_version
 if [ -n "$2" ]; then
   version_to_install="$2"
 fi
 
-if [ "$version_to_install" != "1.72.0" ]; then
+if [ "$version_to_install" != "1.86.0" ]; then
   echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
   version_to_install=$default_version
 fi
 
-if [ "$version_to_install" == "1.72.0" ]; then
+if [ "$version_to_install" == "1.86.0" ]; then
   # hadolint ignore=DL3003
   mkdir -p /opt/boost-library &&
-    curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 &&
-    mv boost_1_72_0.tar.bz2 /opt/boost-library &&
+    curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download >boost_1_86_0.tar.bz2 &&
+    mv boost_1_86_0.tar.bz2 /opt/boost-library &&
     cd /opt/boost-library &&
-    tar --bzip2 -xf boost_1_72_0.tar.bz2 &&
-    cd /opt/boost-library/boost_1_72_0 &&
+    tar --bzip2 -xf boost_1_86_0.tar.bz2 &&
+    cd /opt/boost-library/boost_1_86_0 &&
     ./bootstrap.sh --prefix=/usr/ &&
     ./b2 --without-python install &&
     cd /root &&

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -202,9 +202,24 @@ public static Object getExternalHandler() {
   private final boolean fallbackAllowed;
   private final boolean bindToWildCardAddress;
   private final byte[] clientId;
-  private final int maxAsyncCalls;
+  private int maxAsyncCalls;
   private final AtomicInteger asyncCallCounter = new AtomicInteger(0);
 
+  @VisibleForTesting
+  public int getAsyncCallCounter() {
+    return asyncCallCounter.get();
+  }
+
+  @VisibleForTesting
+  public void setMaxAsyncCalls(int limits) {
+    this.maxAsyncCalls = limits;
+  }
+
+  @VisibleForTesting
+  public boolean isAsyncCallCheckEabled() {
+    return maxAsyncCalls >= 0;
+  }
+
   /**
    * set the ping interval value in configuration
    * 
@@ -1460,7 +1475,7 @@ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
   }
 
   private void checkAsyncCall() throws IOException {
-    if (isAsynchronousMode()) {
+    if (isAsynchronousMode() && isAsyncCallCheckEabled()) {
       if (asyncCallCounter.incrementAndGet() > maxAsyncCalls) {
         String errMsg = String.format(
             "Exceeded limit of max asynchronous calls: %d, " +
@@ -1518,7 +1533,7 @@ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
         throw ioe;
       }
     } catch (Exception e) {
-      if (isAsynchronousMode()) {
+      if (isAsynchronousMode() && isAsyncCallCheckEabled()) {
         releaseAsyncCall();
       }
       throw e;
@@ -1527,7 +1542,9 @@ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
     if (isAsynchronousMode()) {
       CompletableFuture<Writable> result = call.rpcResponseFuture.handle(
           (rpcResponse, e) -> {
-            releaseAsyncCall();
+            if (isAsyncCallCheckEabled()) {
+              releaseAsyncCall();
+            }
             if (e != null) {
               IOException ioe = (IOException) e;
               throw new CompletionException(warpIOException(ioe, connection));

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java

@@ -27,12 +27,17 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.attribute.AclEntry;
+import java.nio.file.attribute.AclEntryPermission;
 import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
+import java.nio.file.attribute.UserPrincipal;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Set;
@@ -51,6 +56,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.nio.file.attribute.AclEntryFlag.DIRECTORY_INHERIT;
+import static java.nio.file.attribute.AclEntryFlag.FILE_INHERIT;
+import static java.nio.file.attribute.AclEntryType.ALLOW;
+
 /** Run a Hadoop job jar. */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -286,25 +295,19 @@ public void run(String[] args) throws Throwable {
     }
     mainClassName = mainClassName.replaceAll("/", ".");
 
-    File tmpDir = new File(System.getProperty("java.io.tmpdir"));
-    ensureDirectory(tmpDir);
-
     final File workDir;
     try {
-      FileAttribute<Set<PosixFilePermission>> perms = PosixFilePermissions
-          .asFileAttribute(PosixFilePermissions.fromString("rwx------"));
-      workDir = Files.createTempDirectory(tmpDir.toPath(), "hadoop-unjar", perms).toFile();
+      workDir = createWorkDirectory();
     } catch (IOException | SecurityException e) {
       // If user has insufficient perms to write to tmpDir, default
       // "Permission denied" message doesn't specify a filename.
       System.err.println("Error creating temp dir in java.io.tmpdir "
-                         + tmpDir + " due to " + e.getMessage());
+                         + System.getProperty("java.io.tmpdir") + " due to "
+                         + e.getMessage());
       System.exit(-1);
       return;
     }
 
-    ensureDirectory(workDir);
-
     ShutdownHookManager.get().addShutdownHook(
         new Runnable() {
           @Override
@@ -333,6 +336,55 @@ public void run() {
     }
   }
 
+  static File createWorkDirectory() throws IOException {
+    File tmpDir = new File(System.getProperty("java.io.tmpdir"));
+    ensureDirectory(tmpDir);
+
+    File workDir = Files.createTempDirectory(tmpDir.toPath(), "hadoop-unjar",
+        directoryPermissions()).toFile();
+    ensureDirectory(workDir);
+    return workDir;
+  }
+
+  private static FileAttribute<?> directoryPermissions() throws IOException {
+    Set<String> views = FileSystems.getDefault().supportedFileAttributeViews();
+    if (views.contains("posix")) {
+      return PosixFilePermissions
+          .asFileAttribute(PosixFilePermissions.fromString("rwx------"));
+    } else if (views.contains("acl")) {
+      return userOnly();
+    } else {
+      throw new IOException("unrecognized FileSystem type " +
+          FileSystems.getDefault());
+    }
+  }
+
+  private static FileAttribute<?> userOnly() throws IOException {
+    UserPrincipal user =
+        FileSystems.getDefault()
+            .getUserPrincipalLookupService()
+            .lookupPrincipalByName(System.getProperty("user.name"));
+    List<AclEntry> acl =
+        Collections.singletonList(AclEntry.newBuilder()
+            .setType(ALLOW)
+            .setPrincipal(user)
+            .setPermissions(EnumSet.allOf(AclEntryPermission.class))
+            .setFlags(DIRECTORY_INHERIT, FILE_INHERIT)
+            .build());
+    return
+        new FileAttribute<List<AclEntry>>() {
+          @Override
+          public String name() {
+            return "acl:acl";
+          }
+
+          @Override
+          public List<AclEntry> value() {
+            return acl;
+          }
+        };
+  }
+
   /**
    * Creates a classloader based on the environment that was specified by the
    * user. If HADOOP_USE_CLIENT_CLASSLOADER is specified, it creates an

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2409,6 +2409,7 @@ The switch to turn S3A auditing on or off.
   <value>100</value>
   <description>
     Define the maximum number of outstanding async calls.
+    If negative, there is no limit on the number of outstanding async calls.
   </description>
 </property>