YARN-11823: add new endpoints for getting jstacks of ApplicationId and NodeManager process

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2325,6 +2325,11 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
       "runtime.linux.";
 
+  /** Flag to turn on/off jstack endpoints for NodeManager. By default is False **/
+  public static final String NM_JSTACK_ENDPOINTS_ENABLED =
+          NM_PREFIX + "jstack-endpoints.enabled";
+  public static final boolean DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED = false;
+
   /**
    * Comma separated list of runtimes that are allowed when using
    * LinuxContainerExecutor. The standard values are:

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -5856,6 +5856,17 @@
     <value>50000</value>
   </property>
 
+
+  <property>
+    <description>
+      This configuration will be used to turn on/turn off nodemanager jstack endpoints
+      /ws/v1/node/jstack/{numberOfJStack} and /ws/v1/node/apps/{appid}/jstack/{numberOfJStack}.
+      The purpose is to minimise security risk. By default is set to false.
+    </description>
+    <name>yarn.nodemanager.jstack-endpoints.enabled</name>
+    <value>false</value>
+  </property>
+
   <property>
     <description>
       Set the connect timeout interval, in milliseconds.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java

@@ -0,0 +1,229 @@
+/** * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.Map;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.Arrays;
+
+public class DiagnosticJStackService {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DiagnosticJStackService.class);
+
+  private static final String NM_USER = System.getProperty("user.name");
+  private static final String JSTACK_PATH = System.getProperty("java.home") + "/bin/jstack";
+  private final Context context;
+  private final Configuration conf;
+
+  public DiagnosticJStackService(Context context) {
+    this.context = context;
+    this.conf = context.getConf();
+  }
+
+  public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req)
+      throws IOException {
+    checkShellNotWindows();
+
+    long nodeManagerPid = ProcessHandle.current().pid();
+
+    checkAdminACL(req);
+
+    return runJStack(nodeManagerPid, numberOfJStack);
+  }
+
+  private void checkAdminACL(HttpServletRequest req) throws IOException {
+    UserGroupInformation callerUGI = getUserGroupInformation(req);
+
+    boolean isAdmin = context.getApplicationACLsManager().isAdmin(callerUGI);
+
+    if (!isAdmin) {
+      throw new YarnRuntimeException("User " + callerUGI.getShortUserName() +
+              " is not authorized to run jstack on NodeManager ");
+    }
+  }
+
+  public String collectApplicationThreadDump(
+      String appId, int numberOfJStack, HttpServletRequest req)
+      throws IOException {
+    checkShellNotWindows();
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+
+    Application app = context.getApplications().get(applicationId);
+    if (app == null){
+      throw new YarnRuntimeException("Application " + applicationId + " does not exist");
+    }
+
+    checkApplicationACL(req, app);
+
+    Map<ContainerId, List<Long>> containerPids = getApplicationContainerPids(app);
+
+    return runJStack(containerPids, numberOfJStack);
+  }
+
+  private void checkApplicationACL(HttpServletRequest req, Application app) throws IOException {
+    UserGroupInformation callerUGI = getUserGroupInformation(req);
+
+    boolean isAuthorized = context.getApplicationACLsManager().checkAccess(
+        callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId()
+    );
+
+    if(!isAuthorized){
+      throw new YarnRuntimeException("User " + callerUGI.getShortUserName() +
+        " is not authorized to view application " + app.getAppId());
+    }
+
+  }
+
+  private void checkShellNotWindows() {
+    if (Shell.WINDOWS) {
+      throw new UnsupportedOperationException("Not implemented for Windows.");
+    }
+  }
+
+  protected Map<ContainerId, List<Long>> getApplicationContainerPids(Application app){
+    Map<ContainerId, List<Long>> containerPids = new HashMap<>();
+
+    for (ContainerId containerId : app.getContainers().keySet()){
+      String pidForContainerIdStr = context.getContainerExecutor().getProcessId(containerId);
+      long parentPid = Long.parseLong(pidForContainerIdStr);
+
+      List<Long> javaContainerPids = ProcessHandle.of(parentPid).stream()
+          .flatMap(ProcessHandle::descendants)
+          .filter(childProcess -> {
+            String cmdLine = childProcess.info().commandLine().orElse("").trim();
+            if (cmdLine.isEmpty()){
+              return false;
+            }
+            String executable = cmdLine.split("\\s+")[0];
+            // The first token is always the executable binary
+            return executable.equals("java") || executable.endsWith("/java");
+          })
+          .map(ProcessHandle::pid)
+          .toList();
+
+      containerPids.put(containerId, javaContainerPids);
+
+    }
+
+    LOG.info("Application PIDs by ContainerId: {}", containerPids);
+
+    return containerPids;
+  }
+
+  private String runJStack(Map<ContainerId, List<Long>> containerPids, int numJStacks){
+    StringBuilder result = new StringBuilder();
+
+    for(Map.Entry<ContainerId, List<Long>> entry : containerPids.entrySet()){
+      ContainerId containerId = entry.getKey();
+      List<Long> javaContainerPids = entry.getValue();
+
+      if (javaContainerPids.isEmpty()){
+        result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " +
+            "because no Java Process ID exist ===", containerId.toString()));
+      } else {
+        for (Long pid : javaContainerPids) {
+          result.append(String.format(
+              "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n",
+              containerId.toString(), pid, runJStack(pid, numJStacks)));
+        }
+      }
+
+    }
+
+    return result.toString();
+  }
+
+  private String runJStack(long pid, int numJStacks) {
+    Optional<ProcessHandle> processHandleOpt = ProcessHandle.of(pid);
+
+    if (processHandleOpt.isEmpty()){
+      String msg = String.format("Process with PID " + pid + " is no longer exists");
+      LOG.warn(msg);
+      return "Status: Skipped Process with PID " + msg;
+    }
+
+    ProcessHandle processHandle = processHandleOpt.get();
+
+    String runningUser = processHandle.info().user().orElse(NM_USER);
+    String containerExecutorPath =
+        PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf);
+
+    String[] jstackCommand = {
+        containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH
+    };
+
+    LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand));
+
+    StringBuilder result = new StringBuilder();
+
+    for (int i = 0; i < numJStacks; i++) {
+      Shell.ShellCommandExecutor cmd =
+          new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000);
+
+      try {
+        cmd.execute();
+        result.append(String.format(
+            "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput()));
+      } catch (IOException e) {
+        result.append(String.format(
+            "Failed to run jstack on PID: " + pid + " at iteration: " + i +
+            " (Process likely exited before/during running jstack): " + e.getMessage()));
+        break;
+      }
+    }
+
+    return result.toString();
+  }
+
+  private UserGroupInformation getUserGroupInformation(HttpServletRequest req) throws IOException {
+    String remoteUser = req.getRemoteUser();
+    UserGroupInformation callerUGI;
+
+    if (remoteUser != null) {
+      callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+    } else {
+      callerUGI = UserGroupInformation.getCurrentUser(); // Fallback to current OS user
+    }
+
+    LOG.info("Checking ACL for Caller UGI: {}", callerUGI.toString());
+
+    return callerUGI;
+
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -31,6 +31,7 @@
 import java.util.Set;
 
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
@@ -107,7 +108,9 @@ public class NMWebServices {
   private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
   private String redirectWSUrl;
   private LogAggregationFileControllerFactory factory;
+  private DiagnosticJStackService diagnosticJStackService;
   private boolean filterAppsByUser = false;
+  private boolean isJStackEndpointsEnable = false;
 
   @javax.ws.rs.core.Context
   private HttpServletRequest request;
@@ -132,6 +135,11 @@ public NMWebServices(final @javax.inject.Named("nm") Context nm,
     this.filterAppsByUser = this.nmContext.getConf().getBoolean(
         YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
         YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+    this.isJStackEndpointsEnable = this.nmContext.getConf().getBoolean(
+            YarnConfiguration.NM_JSTACK_ENDPOINTS_ENABLED,
+            YarnConfiguration.DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED
+    );
+    this.diagnosticJStackService = new DiagnosticJStackService(this.nmContext);
   }
 
   public NMWebServices(final Context nm, final ResourceView view,
@@ -628,6 +636,68 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context
     return Response.ok().build();
   }
 
+
+  @GET
+  @Path("/jstack/{numberOfJStack}")
+  @Produces({ MediaType.TEXT_PLAIN})
+  public Response getNodeThreadDump(@javax.ws.rs.core.Context HttpServletRequest req,
+      @PathParam("numberOfJStack") int numberOfJStack) {
+    if (!isJStackEndpointsEnable) {
+      return Response.status(Status.METHOD_NOT_ALLOWED)
+              .build();
+    }
+
+    try {
+      return Response.status(Status.OK)
+              .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack, req))
+              .build();
+    } catch (YarnRuntimeException e) {
+      return Response.status(Status.FORBIDDEN)
+              .entity(e.getMessage())
+              .build();
+    } catch (IOException e){
+      return Response.status(Status.INTERNAL_SERVER_ERROR)
+              .entity("Shell command has failed: " + e.getMessage() + ". " +
+                      "For more information please check the NodeManager logs.")
+              .build();
+    }
+
+  }
+
+
+  @GET
+  @Path("/apps/{appid}/jstack/{numberOfJStack}")
+  @Produces({MediaType.TEXT_PLAIN})
+  public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletRequest req,
+      @PathParam("appid") String appId,
+      @PathParam("numberOfJStack") int numberOfJStack) {
+    if (!isJStackEndpointsEnable) {
+      return Response.status(Status.METHOD_NOT_ALLOWED)
+              .build();
+    }
+
+    try {
+      return Response.status(Status.OK)
+              .entity(diagnosticJStackService
+                      .collectApplicationThreadDump(appId, numberOfJStack, req))
+              .build();
+    } catch (IllegalArgumentException e){
+      return Response.status(Status.BAD_REQUEST)
+              .entity("The applicationId is invalid: " + appId + ". " + e.getMessage())
+              .build();
+    } catch (YarnRuntimeException e) {
+      return Response.status(Status.FORBIDDEN)
+              .entity(e.getMessage())
+              .build();
+    } catch (IOException e){
+      return Response.status(Status.INTERNAL_SERVER_ERROR)
+              .entity("Shell command has failed: " + e.getMessage() + ". " +
+                "For more information please check the NodeManager logs.")
+              .build();
+    }
+
+  }
+
   private long parseLongParam(String bytes) {
     if (bytes == null || bytes.isEmpty()) {
       return Long.MAX_VALUE;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c

@@ -3310,3 +3310,16 @@ int remove_docker_container(char**argv, int argc) {
   }
   return exit_code;
 }
+
+int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path) {
+  int exit_code = set_user(user);
+  if (exit_code != 0) {
+    fprintf(ERRORFILE, "Failed to set user to %s\n", user);
+    return exit_code;
+  }
+
+  execl(jstack_path, "jstack", pid, (char *) NULL);
+
+  fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno));
+  return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT;
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h

@@ -54,7 +54,8 @@ enum operations {
   RUN_AS_USER_SYNC_YARN_SYSFS = 15,
   EXEC_CONTAINER = 16,
   RUN_RUNC_CONTAINER = 17,
-  REAP_RUNC_LAYER_MOUNTS = 18
+  REAP_RUNC_LAYER_MOUNTS = 18,
+  RUN_JSTACK = 19
 };
 
 #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -368,3 +369,8 @@ int is_terminal_support_enabled();
  * Check if runC feature is enabled
  */
 int is_runc_support_enabled();
+
+/**
+ * Run jstack as a specific user.
+ */
+int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path);