From 80814ffcab08e6ffc384e492b2411f4d2a6a0ceb Mon Sep 17 00:00:00 2001 From: Hean Chhinling Date: Thu, 5 Jun 2025 17:09:43 +0200 Subject: [PATCH 01/35] YARN-11823: add new endpoints for getting jstacks of application and nodes --- .../nodemanager/DiagnosticJStackService.java | 134 ++++++++++++++++++ .../nodemanager/webapp/NMWebServices.java | 31 ++++ .../resources/diagnostics/jstack_collector.py | 95 +++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java new file mode 100644 index 00000000000000..e1fef62e316694 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java @@ -0,0 +1,134 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DiagnosticJStackService { + + private static final Logger LOG = LoggerFactory + .getLogger(DiagnosticJStackService.class); + private static final String PYTHON_COMMAND = "python3"; + private static String scriptLocation = null; + + static { + try { + // Extract script from JAR to a temp file + InputStream in = DiagnosticJStackService.class.getClassLoader() + .getResourceAsStream("diagnostics/jstack_collector.py"); + File tempScript = File.createTempFile("jstack_collector", ".py"); + Files.copy(in, tempScript.toPath(), StandardCopyOption.REPLACE_EXISTING); + tempScript.setExecutable(true); // Set execute permission + scriptLocation = tempScript.getAbsolutePath(); + } catch (IOException e) { + LOG.error("Failed to extract Python script from JAR", e); + } + } + + public static String collectNodeJStack() + throws Exception { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows"); + } + + ProcessBuilder pb = createProcessBuilder(); + + return executeCommand(pb); + + } + + + + public static String collectAppJStack(String appId) + throws Exception { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); + } + ProcessBuilder pb = createProcessBuilder(appId); + + LOG.info("Diagnostic process environment: {}", pb.environment()); + + return executeCommand(pb); + } + + protected static ProcessBuilder createProcessBuilder() { + List commandList = + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation)); + + return new ProcessBuilder(commandList); + } + + + protected static ProcessBuilder createProcessBuilder(String appId) { + List commandList = + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, appId)); + + return new ProcessBuilder(commandList); + } + + private static String executeCommand(ProcessBuilder pb) + throws Exception { + Process process = pb.start(); + int exitCode; + StringBuilder outputBuilder = new StringBuilder(); + StringBuilder errorBuilder = new StringBuilder(); + + try ( + BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(process.getInputStream(), + StandardCharsets.UTF_8)); + BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream(), + StandardCharsets.UTF_8)); + ) { + + String line; + while ((line = stdoutReader.readLine()) != null) { + outputBuilder.append(line).append("\n"); + } + + while ((line = stderrReader.readLine()) != null) { + errorBuilder.append(line).append("\n"); + } + if (!errorBuilder.toString().isEmpty()) { + LOG.error("Python script stderr: {}", errorBuilder); + } + + process.waitFor(); + } catch (Exception e) { + LOG.error("Error getting JStack: {}", pb.command()); + throw e; + } + exitCode = process.exitValue(); + if (exitCode != 0) { + throw new IOException("The JStack collector script exited with non-zero " + + "exit code: " + exitCode); + } + + return outputBuilder.toString(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 6cfd43cd13acd4..5b8464ee2fcd84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -31,12 +31,14 @@ import java.util.Set; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.yarn.server.nodemanager.DiagnosticJStackService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -271,6 +273,35 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context } + @GET + @Path("/jstack") + @Produces({MediaType.TEXT_PLAIN}) + public Response getNodeJStack() { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectNodeJStack()) // Make sure the NodeManager have python3 install + .build(); + } catch (Exception e) { + throw new WebAppException("Error collection NodeManager JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } + + + @GET + @Path("/apps/{appid}/jstack") + @Produces({MediaType.TEXT_PLAIN}) + public Response getApplicationJStack(@PathParam("appid") String appId) { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectAppJStack(appId)) // Make sure the NodeManager have python3 install + .build(); + } catch (Exception e) { + throw new WebAppException("Error collecting Application JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } + /** * Returns log file's name as well as current file size for a container. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py new file mode 100644 index 00000000000000..a17893cecad4a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys + +NUMBER_OF_JSTACK = 3 + +def get_nodemanager_pid(): + results = run_command("ps aux | grep nodemanager | grep -v grep") + # ps aux | grep nodemanager | grep -v grep + # root 414 1.3 1.7 8124480 434520 ? Sl 11:36 0:52 /usr/lib/jvm/java-8-openjdk//bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/opt/hadoop/logs -Dyarn.log.file=hadoop.log -Dyarn.home.dir=/opt/hadoop -Dyarn.root.logger=INFO,console -Dhadoop.log.dir=/opt/hadoop/logs -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=/opt/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED --add-opens=java.base/sun.security.util=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED org.apache.hadoop.yarn.server.nodemanager.NodeManager + pids = [] # Some host may contain more than one NodeManager + for result in results.strip().splitlines(): + pid = result.split()[1] + pids.append(pid) + + return pids + + +def get_app_pid(app_id): + + # results= ''' + # root 413 1.7 2.0 8355580 512972 ? Sl 11:21 2:56 /usr/lib/jvm/java-8-openjdk//bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dhadoop.log.dir=/opt/hadoop/logs -Dhadoop.log.file=NODEMANAGER.log -Dyarn.log.dir=/opt/hadoop/logs -Dyarn.log.file=NODEMANAGER.log -Dyarn.home.dir=/opt/hadoop -Dyarn.root.logger=INFO,DRFA -Dhadoop.home.dir=/opt/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED --add-opens=java.base/sun.security.util=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED --enable-native-access=ALL-UNNAMED org.apache.hadoop.yarn.server.nodemanager.NodeManager + # root 41611 4.1 1.9 2414568 470660 ? Sl 14:08 0:16 /usr/lib/jvm/java-8-openjdk//bin/java -Xmx750m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_type GUARANTEED --container_memory 750 --container_vcores 1 --num_containers 500 --priority 0 --appname DistributedShell --homedir hdfs://namenode:9000/user/root + # ''' + results = run_command("ps aux | grep jvm/java | grep -v -e /bin/bash -e grep") # TODO: later include "grep app_id" for long java application like mapreduce + pids = [] + for result in results.strip().splitlines(): + pid = result.split()[1] + pids.append(pid) + + return pids + + +def execute_jstack(pids): + all_jstacks = [] + + for pid in pids: + for i in range(NUMBER_OF_JSTACK): # Get multiple jstack + jstack_output = run_command("jstack", pid) + all_jstacks.append("--- JStack iteration-{} for PID: {} ---\n{}".format(i, pid, jstack_output)) + + return "\n".join(all_jstacks) + + +def run_command(*argv): + try: + cmd = " ".join(arg for arg in argv) + print("Running command with arguments:", cmd) + response = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True) + response_str = response.stdout.decode('utf-8') + except subprocess.CalledProcessError as e: + response_str = "Unable to run command: {}".format(e) + print(response_str, file=sys.stderr) + except Exception as e: + response_str = "Exception occurred: {}".format(e) + print(response_str, file=sys.stderr) + + return response_str + + +def main(): + + # app_id = "application_1748517687882_0013" + if len(sys.argv) > 1: + app_id = sys.argv[1] + pids = get_app_pid(app_id) + else: + pids = get_nodemanager_pid() + + if not pids: + print("No active process id in this NodeManager.") + sys.exit(0) + + jstacks = execute_jstack(pids) + print(jstacks) # The Initiated java processBuilder will read this stdout + + +if __name__ == "__main__": + main() + From 86a99ec52ef891d4accf3d6b9102b7208f914b1d Mon Sep 17 00:00:00 2001 From: Hean Chhinling Date: Mon, 16 Jun 2025 12:42:52 +0200 Subject: [PATCH 02/35] Adding number of jstack through rest api --- .../nodemanager/DiagnosticJStackService.java | 16 ++++++++-------- .../server/nodemanager/webapp/NMWebServices.java | 15 +++++++++------ .../resources/diagnostics/jstack_collector.py | 12 +++++++----- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java index e1fef62e316694..bb6fae0eda2a7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java @@ -50,13 +50,13 @@ public class DiagnosticJStackService { } } - public static String collectNodeJStack() + public static String collectNodeThreadDump(String numberOfJStack) throws Exception { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows"); } - ProcessBuilder pb = createProcessBuilder(); + ProcessBuilder pb = createProcessBuilder(numberOfJStack); return executeCommand(pb); @@ -64,29 +64,29 @@ public static String collectNodeJStack() - public static String collectAppJStack(String appId) + public static String collectApplicationThreadDump(String appId, String numberOfJStack) throws Exception { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } - ProcessBuilder pb = createProcessBuilder(appId); + ProcessBuilder pb = createProcessBuilder(appId, numberOfJStack); LOG.info("Diagnostic process environment: {}", pb.environment()); return executeCommand(pb); } - protected static ProcessBuilder createProcessBuilder() { + protected static ProcessBuilder createProcessBuilder(String numberOfJStack) { List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation)); + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, numberOfJStack)); return new ProcessBuilder(commandList); } - protected static ProcessBuilder createProcessBuilder(String appId) { + protected static ProcessBuilder createProcessBuilder(String appId, String numberOfJStack) { List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, appId)); + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, appId, numberOfJStack)); return new ProcessBuilder(commandList); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 5b8464ee2fcd84..5694ab83f7189d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -274,12 +274,13 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context } @GET - @Path("/jstack") + @Path("/jstack/{numberOfJStack}") @Produces({MediaType.TEXT_PLAIN}) - public Response getNodeJStack() { + public Response getNodeThreadDump(@PathParam("numberOfJStack") String numberOfJStack) + { // Make sure the NodeManager have python3 install try { return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectNodeJStack()) // Make sure the NodeManager have python3 install + .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) .build(); } catch (Exception e) { throw new WebAppException("Error collection NodeManager JStack: " + e.getMessage() + ". " + @@ -289,12 +290,14 @@ public Response getNodeJStack() { @GET - @Path("/apps/{appid}/jstack") + @Path("/apps/{appid}/jstack/{numberOfJStack}") @Produces({MediaType.TEXT_PLAIN}) - public Response getApplicationJStack(@PathParam("appid") String appId) { + public Response getApplicationJStack(@PathParam("appid") String appId, + @PathParam("numberOfJStack") String numberOfJStack) + { // Make sure the NodeManager have python3 install try { return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectAppJStack(appId)) // Make sure the NodeManager have python3 install + .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) .build(); } catch (Exception e) { throw new WebAppException("Error collecting Application JStack: " + e.getMessage() + ". " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py index a17893cecad4a9..3559d22efb4ea0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py @@ -17,7 +17,6 @@ import subprocess import sys -NUMBER_OF_JSTACK = 3 def get_nodemanager_pid(): results = run_command("ps aux | grep nodemanager | grep -v grep") @@ -46,11 +45,11 @@ def get_app_pid(app_id): return pids -def execute_jstack(pids): +def execute_jstack(pids, number_of_jstack): all_jstacks = [] for pid in pids: - for i in range(NUMBER_OF_JSTACK): # Get multiple jstack + for i in range(number_of_jstack): # Get multiple jstack jstack_output = run_command("jstack", pid) all_jstacks.append("--- JStack iteration-{} for PID: {} ---\n{}".format(i, pid, jstack_output)) @@ -76,17 +75,20 @@ def run_command(*argv): def main(): # app_id = "application_1748517687882_0013" - if len(sys.argv) > 1: + + if "app" in sys.argv[0] > 1: app_id = sys.argv[1] + number_of_jstack= sys.argv[2] pids = get_app_pid(app_id) else: pids = get_nodemanager_pid() + number_of_jstack = sys.argv[1] if not pids: print("No active process id in this NodeManager.") sys.exit(0) - jstacks = execute_jstack(pids) + jstacks = execute_jstack(pids, number_of_jstack) print(jstacks) # The Initiated java processBuilder will read this stdout From 3709a54e70bc3f5d002adb1cbc7c2c987434c2fe Mon Sep 17 00:00:00 2001 From: Hean Chhinling Date: Mon, 16 Jun 2025 22:54:19 +0200 Subject: [PATCH 03/35] Fix the integer data type and remove the plain text output to allow exception response in json format --- .../hadoop/yarn/server/nodemanager/webapp/NMWebServices.java | 1 - .../src/main/resources/diagnostics/jstack_collector.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 5694ab83f7189d..ddee07c3c154d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -275,7 +275,6 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context @GET @Path("/jstack/{numberOfJStack}") - @Produces({MediaType.TEXT_PLAIN}) public Response getNodeThreadDump(@PathParam("numberOfJStack") String numberOfJStack) { // Make sure the NodeManager have python3 install try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py index 3559d22efb4ea0..ce215ecc377d1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py @@ -78,11 +78,11 @@ def main(): if "app" in sys.argv[0] > 1: app_id = sys.argv[1] - number_of_jstack= sys.argv[2] + number_of_jstack= int(sys.argv[2]) pids = get_app_pid(app_id) else: pids = get_nodemanager_pid() - number_of_jstack = sys.argv[1] + number_of_jstack = int(sys.argv[1]) if not pids: print("No active process id in this NodeManager.") From fee2f9efb6e7248249feef4433db0edcc115d0eb Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Wed, 10 Dec 2025 17:55:13 +0700 Subject: [PATCH 04/35] Clean get script location and import each module individually --- .../nodemanager/DiagnosticJStackService.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java index bb6fae0eda2a7c..774bb2bbc00f91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java @@ -21,7 +21,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -34,21 +38,6 @@ public class DiagnosticJStackService { private static final Logger LOG = LoggerFactory .getLogger(DiagnosticJStackService.class); private static final String PYTHON_COMMAND = "python3"; - private static String scriptLocation = null; - - static { - try { - // Extract script from JAR to a temp file - InputStream in = DiagnosticJStackService.class.getClassLoader() - .getResourceAsStream("diagnostics/jstack_collector.py"); - File tempScript = File.createTempFile("jstack_collector", ".py"); - Files.copy(in, tempScript.toPath(), StandardCopyOption.REPLACE_EXISTING); - tempScript.setExecutable(true); // Set execute permission - scriptLocation = tempScript.getAbsolutePath(); - } catch (IOException e) { - LOG.error("Failed to extract Python script from JAR", e); - } - } public static String collectNodeThreadDump(String numberOfJStack) throws Exception { @@ -78,7 +67,7 @@ public static String collectApplicationThreadDump(String appId, String numberOfJ protected static ProcessBuilder createProcessBuilder(String numberOfJStack) { List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, numberOfJStack)); + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), numberOfJStack)); return new ProcessBuilder(commandList); } @@ -86,7 +75,7 @@ protected static ProcessBuilder createProcessBuilder(String numberOfJStack) { protected static ProcessBuilder createProcessBuilder(String appId, String numberOfJStack) { List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation, appId, numberOfJStack)); + new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), appId, numberOfJStack)); return new ProcessBuilder(commandList); } @@ -131,4 +120,19 @@ private static String executeCommand(ProcessBuilder pb) return outputBuilder.toString(); } + private static String getScriptLocation() { + try { + // Extract script from JAR to a temp file + InputStream in = DiagnosticJStackService.class.getClassLoader() + .getResourceAsStream("diagnostics/jstack_collector.py"); + File tempScript = File.createTempFile("jstack_collector", ".py"); + Files.copy(in, tempScript.toPath(), StandardCopyOption.REPLACE_EXISTING); + tempScript.setExecutable(true); // Set execute permission + return tempScript.getAbsolutePath(); + } catch (IOException e) { + LOG.error("Failed to extract Python script from JAR", e); + return null; + } + } + } From 38c757f49e36b41002efb66993a711fa3ef14981 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Wed, 10 Dec 2025 22:41:00 +0700 Subject: [PATCH 05/35] Add Unit-test for the new diagnostic service class --- .../TestDiagnosticJStackService.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java new file mode 100644 index 00000000000000..aea74b1ee57cc1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java @@ -0,0 +1,52 @@ +package org.apache.hadoop.yarn.server.nodemanager; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestDiagnosticJStackService { + + @Test + public void testScriptLocationShouldReturnExistingExecutableFile(){ + String scriptPath = callPrivateGetScriptLocation(); + assertNotNull(scriptPath, "Script location should not be null"); + } + + @Test + public void testCreateProcessBuilderAppCommandNumJStackOption(){ + ProcessBuilder pb = DiagnosticJStackService + .createProcessBuilder("5"); + + List cmd = pb.command(); + assertEquals(3, cmd.size()); + assertEquals("python3", cmd.get(0)); + assertTrue(cmd.get(1).contains("jstack_collector"), "Script path should contain jstack_collector"); + assertEquals("5", cmd.get(2)); + } + + @Test + public void testCreateProcessBuilderAppCommandAppIdJStackOption(){ + ProcessBuilder pb = DiagnosticJStackService + .createProcessBuilder("app_123", "5"); + + List cmd = pb.command(); + assertEquals(4, cmd.size()); + assertEquals("python3", cmd.get(0)); + assertTrue(cmd.get(1).contains("jstack_collector"), "Script path should contain jstack_collector"); + assertEquals("app_123", cmd.get(2)); + assertEquals("5", cmd.get(3)); + } + + private String callPrivateGetScriptLocation() { + try { + Method m = DiagnosticJStackService.class.getDeclaredMethod("getScriptLocation"); + m.setAccessible(true); + return (String) m.invoke(null); + } catch (Exception e){ + throw new RuntimeException(e); + } + } +} From 0157b0f9628e03d9a3c8dec68ae19a7bf4ebfa6d Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 2 Feb 2026 18:47:51 +0100 Subject: [PATCH 06/35] YARN-11823: fix the script for collection jstack application --- .../{ => webapp}/DiagnosticJStackService.java | 6 ++--- .../nodemanager/webapp/NMWebServices.java | 1 - .../resources/diagnostics/jstack_collector.py | 2 +- .../TestDiagnosticJStackService.java | 23 ++++++++++++++++++- 4 files changed, 26 insertions(+), 6 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/{ => webapp}/DiagnosticJStackService.java (95%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 774bb2bbc00f91..c0f6839d3c5b9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager; +package org.apache.hadoop.yarn.server.nodemanager.webapp; import org.apache.hadoop.util.Shell; import org.slf4j.Logger; @@ -65,7 +65,7 @@ public static String collectApplicationThreadDump(String appId, String numberOfJ return executeCommand(pb); } - protected static ProcessBuilder createProcessBuilder(String numberOfJStack) { + public static ProcessBuilder createProcessBuilder(String numberOfJStack) { List commandList = new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), numberOfJStack)); @@ -73,7 +73,7 @@ protected static ProcessBuilder createProcessBuilder(String numberOfJStack) { } - protected static ProcessBuilder createProcessBuilder(String appId, String numberOfJStack) { + public static ProcessBuilder createProcessBuilder(String appId, String numberOfJStack) { List commandList = new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), appId, numberOfJStack)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index ddee07c3c154d6..2bcda8e36b064f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -31,7 +31,6 @@ import java.util.Set; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.yarn.server.nodemanager.DiagnosticJStackService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py index ce215ecc377d1d..f304a4ebfba3c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py @@ -76,7 +76,7 @@ def main(): # app_id = "application_1748517687882_0013" - if "app" in sys.argv[0] > 1: + if sys.argv[1].startswith("application_"): app_id = sys.argv[1] number_of_jstack= int(sys.argv[2]) pids = get_app_pid(app_id) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java index aea74b1ee57cc1..06c6b3605dfc9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java @@ -1,11 +1,32 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.yarn.server.nodemanager; +import org.apache.hadoop.yarn.server.nodemanager.webapp.DiagnosticJStackService; import org.junit.jupiter.api.Test; import java.lang.reflect.Method; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + public class TestDiagnosticJStackService { From 0a3300c49841d50e3d2b25dc3a6ce3a1d7f6ef38 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 5 Feb 2026 17:37:35 +0100 Subject: [PATCH 07/35] YARN-11823: use shellExecutor remove python script and guard the endpoints to enable or disable --- .../hadoop/yarn/conf/YarnConfiguration.java | 5 + .../webapp/DiagnosticJStackService.java | 138 +++++++++--------- .../nodemanager/webapp/NMWebServices.java | 50 ++++--- .../resources/diagnostics/jstack_collector.py | 97 ------------ .../TestDiagnosticJStackService.java | 60 ++++---- 5 files changed, 132 insertions(+), 218 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bdeabcaa93478a..24e73ad79487a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2315,6 +2315,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; + /** Flag to turn on/off jstack endpoints for NodeManager. By default is True **/ + public static final String NM_JSTACK_ENDPOINTS_ENABLED = + NM_PREFIX + "jstack-endpoints.enabled"; + public static final boolean DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED = true; + /** * Comma separated list of runtimes that are allowed when using * LinuxContainerExecutor. The standard values are: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index c0f6839d3c5b9e..1f98c7da889891 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -21,118 +21,114 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; + import java.io.IOException; -import java.io.InputStreamReader; -import java.io.BufferedReader; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class DiagnosticJStackService { private static final Logger LOG = LoggerFactory .getLogger(DiagnosticJStackService.class); - private static final String PYTHON_COMMAND = "python3"; - public static String collectNodeThreadDump(String numberOfJStack) + public static String collectNodeThreadDump(int numberOfJStack) throws Exception { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows"); } - ProcessBuilder pb = createProcessBuilder(numberOfJStack); + List nodeManagerPids = getNodeManagerPids(); - return executeCommand(pb); + return runJStack(nodeManagerPids, numberOfJStack); } - public static String collectApplicationThreadDump(String appId, String numberOfJStack) + public static String collectApplicationThreadDump(String appId, int numberOfJStack) throws Exception { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } - ProcessBuilder pb = createProcessBuilder(appId, numberOfJStack); - - LOG.info("Diagnostic process environment: {}", pb.environment()); + List applicationPids = getApplicationPids(appId); - return executeCommand(pb); + return runJStack(applicationPids, numberOfJStack); } - public static ProcessBuilder createProcessBuilder(String numberOfJStack) { - List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), numberOfJStack)); - return new ProcessBuilder(commandList); + public static List getNodeManagerPids() throws IOException { + Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( + new String[]{ + "bash", + "-c", + "ps aux | grep nodemanager | grep -v grep" + }, + null, + null, + 10_000 + ); + cmd.execute(); + return extractPids(cmd.getOutput()); } + public static List getApplicationPids(String appId) throws IOException { + String psCmd = "ps aux | grep jvm/java | grep " + appId + " | grep -v -e /bin/bash -e grep"; - public static ProcessBuilder createProcessBuilder(String appId, String numberOfJStack) { - List commandList = - new ArrayList<>(Arrays.asList(PYTHON_COMMAND, getScriptLocation(), appId, numberOfJStack)); + Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( + new String[]{ "bash", "-c", psCmd}, + null, + null, + 10_000 + ); - return new ProcessBuilder(commandList); + cmd.execute(); + return extractPids(cmd.getOutput()); } - private static String executeCommand(ProcessBuilder pb) - throws Exception { - Process process = pb.start(); - int exitCode; - StringBuilder outputBuilder = new StringBuilder(); - StringBuilder errorBuilder = new StringBuilder(); - - try ( - BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(process.getInputStream(), - StandardCharsets.UTF_8)); - BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream(), - StandardCharsets.UTF_8)); - ) { - - String line; - while ((line = stdoutReader.readLine()) != null) { - outputBuilder.append(line).append("\n"); - } + public static List extractPids(String psOutput) { - while ((line = stderrReader.readLine()) != null) { - errorBuilder.append(line).append("\n"); - } - if (!errorBuilder.toString().isEmpty()) { - LOG.error("Python script stderr: {}", errorBuilder); - } + LOG.info("Process output: " + psOutput); - process.waitFor(); - } catch (Exception e) { - LOG.error("Error getting JStack: {}", pb.command()); - throw e; - } - exitCode = process.exitValue(); - if (exitCode != 0) { - throw new IOException("The JStack collector script exited with non-zero " + - "exit code: " + exitCode); + List pids = new ArrayList<>(); + for(String line : psOutput.split("\n")) { + // root 414 1.3 1.7 8124480 434520 ? Sl 11:36 + String [] parts = line.trim().split("\\s+"); + if (parts.length > 1){ + pids.add(parts[1]); + } } - return outputBuilder.toString(); + return pids; } - private static String getScriptLocation() { - try { - // Extract script from JAR to a temp file - InputStream in = DiagnosticJStackService.class.getClassLoader() - .getResourceAsStream("diagnostics/jstack_collector.py"); - File tempScript = File.createTempFile("jstack_collector", ".py"); - Files.copy(in, tempScript.toPath(), StandardCopyOption.REPLACE_EXISTING); - tempScript.setExecutable(true); // Set execute permission - return tempScript.getAbsolutePath(); - } catch (IOException e) { - LOG.error("Failed to extract Python script from JAR", e); - return null; + + public static String runJStack(List pids, int numJStacks) throws IOException { + StringBuilder result = new StringBuilder(); + + for(String pid : pids){ + + Shell.ShellCommandExecutor cmd = + new Shell.ShellCommandExecutor( + new String[]{"jstack", pid}, + null, + null, + 60_000 + ); + + for (int i = 0; i < numJStacks; i++) { + cmd.execute(); + + result.append("--- JStack iteration -") + .append(i) + .append(" for PID: ") + .append(pid) + .append("---\n") + .append(cmd.getOutput()) + .append("\n"); + } } + + return result.toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 2bcda8e36b064f..fac8ab63c165ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -109,6 +109,7 @@ public class NMWebServices { private String redirectWSUrl; private LogAggregationFileControllerFactory factory; private boolean filterAppsByUser = false; + private boolean isJStackEndpointsEnable = true; @javax.ws.rs.core.Context private HttpServletRequest request; @@ -133,6 +134,10 @@ public NMWebServices(final @javax.inject.Named("nm") Context nm, this.filterAppsByUser = this.nmContext.getConf().getBoolean( YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.isJStackEndpointsEnable = this.nmContext.getConf().getBoolean( + YarnConfiguration.NM_JSTACK_ENDPOINTS_ENABLED, + YarnConfiguration.DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED + ); } public NMWebServices(final Context nm, final ResourceView view, @@ -274,15 +279,20 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context @GET @Path("/jstack/{numberOfJStack}") - public Response getNodeThreadDump(@PathParam("numberOfJStack") String numberOfJStack) - { // Make sure the NodeManager have python3 install - try { - return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) + { + if (isJStackEndpointsEnable) { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + .build(); + } catch (Exception e) { + throw new WebAppException("Error collection NodeManager JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } else { + return Response.status(Status.METHOD_NOT_ALLOWED) .build(); - } catch (Exception e) { - throw new WebAppException("Error collection NodeManager JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); } } @@ -291,16 +301,22 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") String numberOfJS @Path("/apps/{appid}/jstack/{numberOfJStack}") @Produces({MediaType.TEXT_PLAIN}) public Response getApplicationJStack(@PathParam("appid") String appId, - @PathParam("numberOfJStack") String numberOfJStack) - { // Make sure the NodeManager have python3 install - try { - return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) - .build(); - } catch (Exception e) { - throw new WebAppException("Error collecting Application JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); + @PathParam("numberOfJStack") int numberOfJStack) + { + if (isJStackEndpointsEnable) { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) + .build(); + } catch (Exception e) { + throw new WebAppException("Error collecting Application JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } else { + return Response.status(Status.METHOD_NOT_ALLOWED) + .build(); } + } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py deleted file mode 100644 index f304a4ebfba3c2..00000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import sys - - -def get_nodemanager_pid(): - results = run_command("ps aux | grep nodemanager | grep -v grep") - # ps aux | grep nodemanager | grep -v grep - # root 414 1.3 1.7 8124480 434520 ? Sl 11:36 0:52 /usr/lib/jvm/java-8-openjdk//bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/opt/hadoop/logs -Dyarn.log.file=hadoop.log -Dyarn.home.dir=/opt/hadoop -Dyarn.root.logger=INFO,console -Dhadoop.log.dir=/opt/hadoop/logs -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=/opt/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED --add-opens=java.base/sun.security.util=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED org.apache.hadoop.yarn.server.nodemanager.NodeManager - pids = [] # Some host may contain more than one NodeManager - for result in results.strip().splitlines(): - pid = result.split()[1] - pids.append(pid) - - return pids - - -def get_app_pid(app_id): - - # results= ''' - # root 413 1.7 2.0 8355580 512972 ? Sl 11:21 2:56 /usr/lib/jvm/java-8-openjdk//bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dhadoop.log.dir=/opt/hadoop/logs -Dhadoop.log.file=NODEMANAGER.log -Dyarn.log.dir=/opt/hadoop/logs -Dyarn.log.file=NODEMANAGER.log -Dyarn.home.dir=/opt/hadoop -Dyarn.root.logger=INFO,DRFA -Dhadoop.home.dir=/opt/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED --add-opens=java.base/sun.security.util=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED --enable-native-access=ALL-UNNAMED org.apache.hadoop.yarn.server.nodemanager.NodeManager - # root 41611 4.1 1.9 2414568 470660 ? Sl 14:08 0:16 /usr/lib/jvm/java-8-openjdk//bin/java -Xmx750m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_type GUARANTEED --container_memory 750 --container_vcores 1 --num_containers 500 --priority 0 --appname DistributedShell --homedir hdfs://namenode:9000/user/root - # ''' - results = run_command("ps aux | grep jvm/java | grep -v -e /bin/bash -e grep") # TODO: later include "grep app_id" for long java application like mapreduce - pids = [] - for result in results.strip().splitlines(): - pid = result.split()[1] - pids.append(pid) - - return pids - - -def execute_jstack(pids, number_of_jstack): - all_jstacks = [] - - for pid in pids: - for i in range(number_of_jstack): # Get multiple jstack - jstack_output = run_command("jstack", pid) - all_jstacks.append("--- JStack iteration-{} for PID: {} ---\n{}".format(i, pid, jstack_output)) - - return "\n".join(all_jstacks) - - -def run_command(*argv): - try: - cmd = " ".join(arg for arg in argv) - print("Running command with arguments:", cmd) - response = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True) - response_str = response.stdout.decode('utf-8') - except subprocess.CalledProcessError as e: - response_str = "Unable to run command: {}".format(e) - print(response_str, file=sys.stderr) - except Exception as e: - response_str = "Exception occurred: {}".format(e) - print(response_str, file=sys.stderr) - - return response_str - - -def main(): - - # app_id = "application_1748517687882_0013" - - if sys.argv[1].startswith("application_"): - app_id = sys.argv[1] - number_of_jstack= int(sys.argv[2]) - pids = get_app_pid(app_id) - else: - pids = get_nodemanager_pid() - number_of_jstack = int(sys.argv[1]) - - if not pids: - print("No active process id in this NodeManager.") - sys.exit(0) - - jstacks = execute_jstack(pids, number_of_jstack) - print(jstacks) # The Initiated java processBuilder will read this stdout - - -if __name__ == "__main__": - main() - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java index 06c6b3605dfc9a..7cdcce8e7f383e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java @@ -20,54 +20,48 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.DiagnosticJStackService; import org.junit.jupiter.api.Test; -import java.lang.reflect.Method; + import java.util.List; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; public class TestDiagnosticJStackService { + + @Test - public void testScriptLocationShouldReturnExistingExecutableFile(){ - String scriptPath = callPrivateGetScriptLocation(); - assertNotNull(scriptPath, "Script location should not be null"); + public void testExtractPidsFromEmptyProcessOutput(){ + String psOutput = ""; + + List pids = DiagnosticJStackService.extractPids(psOutput); + + assertTrue(pids.isEmpty()); + } @Test - public void testCreateProcessBuilderAppCommandNumJStackOption(){ - ProcessBuilder pb = DiagnosticJStackService - .createProcessBuilder("5"); - - List cmd = pb.command(); - assertEquals(3, cmd.size()); - assertEquals("python3", cmd.get(0)); - assertTrue(cmd.get(1).contains("jstack_collector"), "Script path should contain jstack_collector"); - assertEquals("5", cmd.get(2)); + public void testExtractPidsFromOneProcessOutput(){ + String psOutput = "root 414 1.3 1.7 8124480 434520 ? Sl 11:36"; + + List pids = DiagnosticJStackService.extractPids(psOutput); + + assertEquals("414", pids.get(0)); + } @Test - public void testCreateProcessBuilderAppCommandAppIdJStackOption(){ - ProcessBuilder pb = DiagnosticJStackService - .createProcessBuilder("app_123", "5"); - - List cmd = pb.command(); - assertEquals(4, cmd.size()); - assertEquals("python3", cmd.get(0)); - assertTrue(cmd.get(1).contains("jstack_collector"), "Script path should contain jstack_collector"); - assertEquals("app_123", cmd.get(2)); - assertEquals("5", cmd.get(3)); - } + public void testExtractPidsFromMultipleProcessOutputs(){ + String psOutput = """ + root 414 1.3 1.7 8124480 434520 ? Sl 11:36 + root 420 1.3 1.7 8124480 434520 ? Sl 11:36 + """; + + List pids = DiagnosticJStackService.extractPids(psOutput); - private String callPrivateGetScriptLocation() { - try { - Method m = DiagnosticJStackService.class.getDeclaredMethod("getScriptLocation"); - m.setAccessible(true); - return (String) m.invoke(null); - } catch (Exception e){ - throw new RuntimeException(e); - } + assertEquals("414", pids.get(0)); + assertEquals("420", pids.get(1)); } + } From 824dfa1fb23031d27353f5fa0343e98e483f846e Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 12 Feb 2026 17:57:36 +0100 Subject: [PATCH 08/35] YARN-11823: update handling exceptions, app_id checks and change the default jstack endpoint to false --- .../hadoop/yarn/conf/YarnConfiguration.java | 4 +- .../src/main/resources/yarn-default.xml | 11 +++ .../webapp/DiagnosticJStackService.java | 12 ++- .../nodemanager/webapp/NMWebServices.java | 98 +++++++++++-------- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 24e73ad79487a0..6b45b473ea7278 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2315,10 +2315,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; - /** Flag to turn on/off jstack endpoints for NodeManager. By default is True **/ + /** Flag to turn on/off jstack endpoints for NodeManager. By default is False **/ public static final String NM_JSTACK_ENDPOINTS_ENABLED = NM_PREFIX + "jstack-endpoints.enabled"; - public static final boolean DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED = true; + public static final boolean DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED = false; /** * Comma separated list of runtimes that are allowed when using diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index efef1e30e3df62..4884f4495e2bd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5854,6 +5854,17 @@ 50000 + + + + This configuration will be used to turn on/turn off nodemanager jstack endpoints + /ws/v1/node/jstack/{numberOfJStack} and /ws/v1/node/apps/{appid}/jstack/{numberOfJStack}. + The purpose is to minimise security risk. By default is set to false. + + yarn.nodemanager.jstack-endpoints.enabled + false + + Set the connect timeout interval, in milliseconds. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 1f98c7da889891..db3583cd4a89d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -31,8 +31,10 @@ public class DiagnosticJStackService { private static final Logger LOG = LoggerFactory .getLogger(DiagnosticJStackService.class); + private DiagnosticJStackService() {} + public static String collectNodeThreadDump(int numberOfJStack) - throws Exception { + throws IOException { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows"); } @@ -46,7 +48,11 @@ public static String collectNodeThreadDump(int numberOfJStack) public static String collectApplicationThreadDump(String appId, int numberOfJStack) - throws Exception { + throws IOException { + if(!appId.matches("application_\\d{13}_\\d{4}")) { + throw new RuntimeException("Invalid application id: " + appId); + } + if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } @@ -61,7 +67,7 @@ public static List getNodeManagerPids() throws IOException { new String[]{ "bash", "-c", - "ps aux | grep nodemanager | grep -v grep" + "ps aux | grep org.apache.hadoop.yarn.server.nodemanager.NodeManager | grep -v grep" }, null, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index fac8ab63c165ab..6512f3cb0c9034 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -109,7 +109,7 @@ public class NMWebServices { private String redirectWSUrl; private LogAggregationFileControllerFactory factory; private boolean filterAppsByUser = false; - private boolean isJStackEndpointsEnable = true; + private boolean isJStackEndpointsEnable = false; @javax.ws.rs.core.Context private HttpServletRequest request; @@ -277,48 +277,6 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context } - @GET - @Path("/jstack/{numberOfJStack}") - public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) - { - if (isJStackEndpointsEnable) { - try { - return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) - .build(); - } catch (Exception e) { - throw new WebAppException("Error collection NodeManager JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } - } else { - return Response.status(Status.METHOD_NOT_ALLOWED) - .build(); - } - } - - - @GET - @Path("/apps/{appid}/jstack/{numberOfJStack}") - @Produces({MediaType.TEXT_PLAIN}) - public Response getApplicationJStack(@PathParam("appid") String appId, - @PathParam("numberOfJStack") int numberOfJStack) - { - if (isJStackEndpointsEnable) { - try { - return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) - .build(); - } catch (Exception e) { - throw new WebAppException("Error collecting Application JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } - } else { - return Response.status(Status.METHOD_NOT_ALLOWED) - .build(); - } - - } - /** * Returns log file's name as well as current file size for a container. * @@ -674,6 +632,60 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context return Response.ok().build(); } + + @GET + @Path("/jstack/{numberOfJStack}") + public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) + { + if (isJStackEndpointsEnable) { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + .build(); + } catch (IOException e){ + throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } catch (Exception e) { + throw new WebAppException( + "Unexpected error collection NodeManager JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } else { + return Response.status(Status.METHOD_NOT_ALLOWED) + .build(); + } + } + + + @GET + @Path("/apps/{appid}/jstack/{numberOfJStack}") + @Produces({MediaType.TEXT_PLAIN}) + public Response getApplicationJStack(@PathParam("appid") String appId, + @PathParam("numberOfJStack") int numberOfJStack) + { + if (isJStackEndpointsEnable) { + try { + return Response.status(Status.OK) + .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) + .build(); + } catch (RuntimeException e){ + throw new WebAppException( + "The applicationId is invalid: " + appId + "." + e.getMessage()); + } catch (IOException e){ + throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } catch (Exception e) { + throw new WebAppException( + "Unexpected error collecting Application JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } else { + return Response.status(Status.METHOD_NOT_ALLOWED) + .build(); + } + + } + private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; From ff7dde623e7d250e195126f1b50a478f4ec77c1a Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 17 Feb 2026 18:20:02 +0100 Subject: [PATCH 09/35] YARN-11823: use processHandle to get NM process ID and fixing the Jstack AppID --- .../webapp/DiagnosticJStackService.java | 96 +++++++++++-------- .../TestDiagnosticJStackService.java | 12 +-- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index db3583cd4a89d2..286cdaaa6d43c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; + import org.apache.hadoop.util.Shell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +26,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class DiagnosticJStackService { @@ -39,9 +44,9 @@ public static String collectNodeThreadDump(int numberOfJStack) throw new UnsupportedOperationException("Not implemented for Windows"); } - List nodeManagerPids = getNodeManagerPids(); + long nodeManagerPid = getNodeManagerPid(); - return runJStack(nodeManagerPids, numberOfJStack); + return runJStack(nodeManagerPid, numberOfJStack); } @@ -56,28 +61,17 @@ public static String collectApplicationThreadDump(String appId, int numberOfJSta if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } - List applicationPids = getApplicationPids(appId); + List applicationPids = getApplicationPids(appId); return runJStack(applicationPids, numberOfJStack); } - public static List getNodeManagerPids() throws IOException { - Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( - new String[]{ - "bash", - "-c", - "ps aux | grep org.apache.hadoop.yarn.server.nodemanager.NodeManager | grep -v grep" - }, - null, - null, - 10_000 - ); - cmd.execute(); - return extractPids(cmd.getOutput()); + public static long getNodeManagerPid() { + return ProcessHandle.current().pid(); } - public static List getApplicationPids(String appId) throws IOException { + public static List getApplicationPids(String appId) throws IOException { String psCmd = "ps aux | grep jvm/java | grep " + appId + " | grep -v -e /bin/bash -e grep"; Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( @@ -91,16 +85,16 @@ public static List getApplicationPids(String appId) throws IOException { return extractPids(cmd.getOutput()); } - public static List extractPids(String psOutput) { + public static List extractPids(String psOutput) { LOG.info("Process output: " + psOutput); - List pids = new ArrayList<>(); + List pids = new ArrayList<>(); for(String line : psOutput.split("\n")) { // root 414 1.3 1.7 8124480 434520 ? Sl 11:36 String [] parts = line.trim().split("\\s+"); if (parts.length > 1){ - pids.add(parts[1]); + pids.add(Long.valueOf(parts[1])); } } @@ -108,33 +102,51 @@ public static List extractPids(String psOutput) { } - public static String runJStack(List pids, int numJStacks) throws IOException { + public static String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); - for(String pid : pids){ - - Shell.ShellCommandExecutor cmd = - new Shell.ShellCommandExecutor( - new String[]{"jstack", pid}, - null, - null, - 60_000 - ); - - for (int i = 0; i < numJStacks; i++) { - cmd.execute(); - - result.append("--- JStack iteration -") - .append(i) - .append(" for PID: ") - .append(pid) - .append("---\n") - .append(cmd.getOutput()) - .append("\n"); - } + for(Long pid : pids){ + result.append(runJStack(pid, numJStacks)); } return result.toString(); } + public static String runJStack(long pid, int numJStacks) throws IOException { + Optional processHandle = ProcessHandle.of(pid); + + if (processHandle.isEmpty()){ + throw new IOException("Process with PID " + pid + " is no longer exists"); + } + + String processOwner = processHandle.get().info().user().orElse("root"); + + + Shell.ShellCommandExecutor cmd = + new Shell.ShellCommandExecutor( + new String[]{"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}, + null, + null, + 60_000 + ); + + StringBuilder result = new StringBuilder(); + + for (int i = 0; i < numJStacks; i++) { + cmd.execute(); + + result.append("--- JStack iteration -") + .append(i) + .append(" for PID: ") + .append(pid) + .append("---\n") + .append(cmd.getOutput()) + .append("\n"); + } + + + return result.toString(); + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java index 7cdcce8e7f383e..2aa339fee7673d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java @@ -35,7 +35,7 @@ public class TestDiagnosticJStackService { public void testExtractPidsFromEmptyProcessOutput(){ String psOutput = ""; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); assertTrue(pids.isEmpty()); @@ -45,9 +45,9 @@ public void testExtractPidsFromEmptyProcessOutput(){ public void testExtractPidsFromOneProcessOutput(){ String psOutput = "root 414 1.3 1.7 8124480 434520 ? Sl 11:36"; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); - assertEquals("414", pids.get(0)); + assertEquals(414, pids.get(0)); } @@ -58,10 +58,10 @@ public void testExtractPidsFromMultipleProcessOutputs(){ root 420 1.3 1.7 8124480 434520 ? Sl 11:36 """; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); - assertEquals("414", pids.get(0)); - assertEquals("420", pids.get(1)); + assertEquals(414, pids.get(0)); + assertEquals(420, pids.get(1)); } } From 7b3ca173b2f31e69c6a4b73e160f4453ee0caa22 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 20 Feb 2026 09:12:32 +0100 Subject: [PATCH 10/35] YARN-11823: update to get process from NMcontext --- .../webapp/DiagnosticJStackService.java | 75 +++++++++++++++---- .../nodemanager/webapp/NMWebServices.java | 4 +- .../TestDiagnosticJStackService.java | 12 +-- 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 286cdaaa6d43c2..169a4b2ebe21a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -18,14 +18,24 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -33,10 +43,14 @@ public class DiagnosticJStackService { + private final Context context; private static final Logger LOG = LoggerFactory .getLogger(DiagnosticJStackService.class); - private DiagnosticJStackService() {} + public DiagnosticJStackService(Context context) { + this.context = context; + } + public static String collectNodeThreadDump(int numberOfJStack) throws IOException { @@ -44,7 +58,7 @@ public static String collectNodeThreadDump(int numberOfJStack) throw new UnsupportedOperationException("Not implemented for Windows"); } - long nodeManagerPid = getNodeManagerPid(); + String nodeManagerPid = getNodeManagerPid(); return runJStack(nodeManagerPid, numberOfJStack); @@ -52,7 +66,7 @@ public static String collectNodeThreadDump(int numberOfJStack) - public static String collectApplicationThreadDump(String appId, int numberOfJStack) + public String collectApplicationThreadDump(String appId, int numberOfJStack) throws IOException { if(!appId.matches("application_\\d{13}_\\d{4}")) { throw new RuntimeException("Invalid application id: " + appId); @@ -61,17 +75,47 @@ public static String collectApplicationThreadDump(String appId, int numberOfJSta if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } - List applicationPids = getApplicationPids(appId); + + + + List applicationPids = getApplicationPids(appId); return runJStack(applicationPids, numberOfJStack); } - public static long getNodeManagerPid() { - return ProcessHandle.current().pid(); + public static String getNodeManagerPid() { + return String.valueOf(ProcessHandle.current().pid()); } - public static List getApplicationPids(String appId) throws IOException { + public List getApplicationPids(String appId) throws IOException { + // List pids = new ArrayList<>(); + + ApplicationId appIdObj = ApplicationId.fromString(appId); + Application app = context.getApplications().get(appIdObj); + if (app != null) { + Map containers = app.getContainers(); + for (ContainerId containerId : containers.keySet()){ + LOG.info("Found container: {}", containerId); + String pidForContainerId = context.getContainerExecutor().getProcessId(containerId); + LOG.info("Parent PID for container: {}", pidForContainerId); + + Optional parentProcess = ProcessHandle.of(Long.parseLong(pidForContainerId)); + parentProcess.ifPresent(processHandle -> processHandle.descendants().forEach( + childProcess -> { + Optional cmd = childProcess.info().command(); + if (cmd.isPresent() && cmd.get().contains("java")) { + LOG.info("Found actual java pid: {}", childProcess.pid()); + } + } + )); + + // pids.add(pidForContainerId); + } + } + + // return pids; + String psCmd = "ps aux | grep jvm/java | grep " + appId + " | grep -v -e /bin/bash -e grep"; Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( @@ -83,18 +127,19 @@ public static List getApplicationPids(String appId) throws IOException { cmd.execute(); return extractPids(cmd.getOutput()); + } - public static List extractPids(String psOutput) { + public static List extractPids(String psOutput) { LOG.info("Process output: " + psOutput); - List pids = new ArrayList<>(); + List pids = new ArrayList<>(); for(String line : psOutput.split("\n")) { // root 414 1.3 1.7 8124480 434520 ? Sl 11:36 String [] parts = line.trim().split("\\s+"); if (parts.length > 1){ - pids.add(Long.valueOf(parts[1])); + pids.add(parts[1]); } } @@ -102,18 +147,18 @@ public static List extractPids(String psOutput) { } - public static String runJStack(List pids, int numJStacks) throws IOException { + public static String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); - for(Long pid : pids){ + for(String pid : pids){ result.append(runJStack(pid, numJStacks)); } return result.toString(); } - public static String runJStack(long pid, int numJStacks) throws IOException { - Optional processHandle = ProcessHandle.of(pid); + public static String runJStack(String pid, int numJStacks) throws IOException { + Optional processHandle = ProcessHandle.of(Long.parseLong(pid)); if (processHandle.isEmpty()){ throw new IOException("Process with PID " + pid + " is no longer exists"); @@ -124,7 +169,7 @@ public static String runJStack(long pid, int numJStacks) throws IOException { Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( - new String[]{"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}, + new String[]{"sudo", "-u", processOwner, "jstack", pid}, null, null, 60_000 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 6512f3cb0c9034..f4169920d0d9aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -665,8 +665,10 @@ public Response getApplicationJStack(@PathParam("appid") String appId, { if (isJStackEndpointsEnable) { try { + DiagnosticJStackService diagnosticJStackService = new DiagnosticJStackService(this.nmContext); + return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) + .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) .build(); } catch (RuntimeException e){ throw new WebAppException( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java index 2aa339fee7673d..7cdcce8e7f383e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java @@ -35,7 +35,7 @@ public class TestDiagnosticJStackService { public void testExtractPidsFromEmptyProcessOutput(){ String psOutput = ""; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); assertTrue(pids.isEmpty()); @@ -45,9 +45,9 @@ public void testExtractPidsFromEmptyProcessOutput(){ public void testExtractPidsFromOneProcessOutput(){ String psOutput = "root 414 1.3 1.7 8124480 434520 ? Sl 11:36"; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); - assertEquals(414, pids.get(0)); + assertEquals("414", pids.get(0)); } @@ -58,10 +58,10 @@ public void testExtractPidsFromMultipleProcessOutputs(){ root 420 1.3 1.7 8124480 434520 ? Sl 11:36 """; - List pids = DiagnosticJStackService.extractPids(psOutput); + List pids = DiagnosticJStackService.extractPids(psOutput); - assertEquals(414, pids.get(0)); - assertEquals(420, pids.get(1)); + assertEquals("414", pids.get(0)); + assertEquals("420", pids.get(1)); } } From a26987c70f68a8ae8c43c88be52ad6e9b4bc6f44 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 20 Feb 2026 12:47:26 +0100 Subject: [PATCH 11/35] YARN-11823: Get the container process id from NodeManager Context --- .../webapp/DiagnosticJStackService.java | 85 ++++++------------- 1 file changed, 25 insertions(+), 60 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 169a4b2ebe21a0..fc4dd7670047e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -18,28 +18,20 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.Context; -import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import javax.inject.Inject; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class DiagnosticJStackService { @@ -58,14 +50,12 @@ public static String collectNodeThreadDump(int numberOfJStack) throw new UnsupportedOperationException("Not implemented for Windows"); } - String nodeManagerPid = getNodeManagerPid(); + long nodeManagerPid = getNodeManagerPid(); return runJStack(nodeManagerPid, numberOfJStack); } - - public String collectApplicationThreadDump(String appId, int numberOfJStack) throws IOException { if(!appId.matches("application_\\d{13}_\\d{4}")) { @@ -76,57 +66,37 @@ public String collectApplicationThreadDump(String appId, int numberOfJStack) throw new UnsupportedOperationException("Not implemented for Windows."); } - - - List applicationPids = getApplicationPids(appId); + List applicationPids = getApplicationPids(appId); return runJStack(applicationPids, numberOfJStack); } - public static String getNodeManagerPid() { - return String.valueOf(ProcessHandle.current().pid()); + public static long getNodeManagerPid() { + return ProcessHandle.current().pid(); } - public List getApplicationPids(String appId) throws IOException { - // List pids = new ArrayList<>(); + public List getApplicationPids(String appId){ + List pids = new ArrayList<>(); ApplicationId appIdObj = ApplicationId.fromString(appId); Application app = context.getApplications().get(appIdObj); if (app != null) { Map containers = app.getContainers(); for (ContainerId containerId : containers.keySet()){ - LOG.info("Found container: {}", containerId); - String pidForContainerId = context.getContainerExecutor().getProcessId(containerId); - LOG.info("Parent PID for container: {}", pidForContainerId); - - Optional parentProcess = ProcessHandle.of(Long.parseLong(pidForContainerId)); - parentProcess.ifPresent(processHandle -> processHandle.descendants().forEach( - childProcess -> { - Optional cmd = childProcess.info().command(); - if (cmd.isPresent() && cmd.get().contains("java")) { - LOG.info("Found actual java pid: {}", childProcess.pid()); - } - } - )); - - // pids.add(pidForContainerId); - } - } + long pidForContainerId = Long.parseLong(context.getContainerExecutor().getProcessId(containerId)); - // return pids; - - String psCmd = "ps aux | grep jvm/java | grep " + appId + " | grep -v -e /bin/bash -e grep"; + ProcessHandle.of(pidForContainerId).ifPresent(parentProcess -> + parentProcess.descendants() + .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) + .map(ProcessHandle::pid) + .forEach(pids::add) + ); - Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( - new String[]{ "bash", "-c", psCmd}, - null, - null, - 10_000 - ); + } + } - cmd.execute(); - return extractPids(cmd.getOutput()); + return pids; } @@ -147,29 +117,29 @@ public static List extractPids(String psOutput) { } - public static String runJStack(List pids, int numJStacks) throws IOException { + public static String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); - for(String pid : pids){ + for(Long pid : pids){ result.append(runJStack(pid, numJStacks)); } return result.toString(); } - public static String runJStack(String pid, int numJStacks) throws IOException { - Optional processHandle = ProcessHandle.of(Long.parseLong(pid)); + public static String runJStack(long pid, int numJStacks) throws IOException { + Optional processHandle = ProcessHandle.of(pid); if (processHandle.isEmpty()){ throw new IOException("Process with PID " + pid + " is no longer exists"); } String processOwner = processHandle.get().info().user().orElse("root"); - + String stringPid = String.valueOf(pid); Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor( - new String[]{"sudo", "-u", processOwner, "jstack", pid}, + new String[]{"sudo", "-u", processOwner, "jstack", stringPid}, null, null, 60_000 @@ -179,14 +149,9 @@ public static String runJStack(String pid, int numJStacks) throws IOException { for (int i = 0; i < numJStacks; i++) { cmd.execute(); - - result.append("--- JStack iteration -") - .append(i) - .append(" for PID: ") - .append(pid) - .append("---\n") - .append(cmd.getOutput()) - .append("\n"); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---\n%s\n", i, pid, cmd.getOutput() + )); } From 0a60eb206edad5a743c1f0cb5a1a5626d3cad332 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 24 Feb 2026 17:24:03 +0100 Subject: [PATCH 12/35] YARN-11823: remove unused method and improve code --- .../webapp/DiagnosticJStackService.java | 32 ++------- .../TestDiagnosticJStackService.java | 67 ------------------- .../webapp/TestDiagnosticJStackService.java | 33 +++++++++ 3 files changed, 40 insertions(+), 92 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index fc4dd7670047e0..5dc292a456a0ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Arrays; public class DiagnosticJStackService { @@ -88,7 +89,8 @@ public List getApplicationPids(String appId){ ProcessHandle.of(pidForContainerId).ifPresent(parentProcess -> parentProcess.descendants() - .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) + .filter(childProcess -> + childProcess.info().command().orElse("").contains("java")) .map(ProcessHandle::pid) .forEach(pids::add) ); @@ -100,22 +102,6 @@ public List getApplicationPids(String appId){ } - public static List extractPids(String psOutput) { - - LOG.info("Process output: " + psOutput); - - List pids = new ArrayList<>(); - for(String line : psOutput.split("\n")) { - // root 414 1.3 1.7 8124480 434520 ? Sl 11:36 - String [] parts = line.trim().split("\\s+"); - if (parts.length > 1){ - pids.add(parts[1]); - } - } - - return pids; - } - public static String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); @@ -135,15 +121,12 @@ public static String runJStack(long pid, int numJStacks) throws IOException { } String processOwner = processHandle.get().info().user().orElse("root"); - String stringPid = String.valueOf(pid); + String[] jstackCommand = {"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}; + + LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); Shell.ShellCommandExecutor cmd = - new Shell.ShellCommandExecutor( - new String[]{"sudo", "-u", processOwner, "jstack", stringPid}, - null, - null, - 60_000 - ); + new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); StringBuilder result = new StringBuilder(); @@ -154,7 +137,6 @@ public static String runJStack(long pid, int numJStacks) throws IOException { )); } - return result.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java deleted file mode 100644 index 7cdcce8e7f383e..00000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDiagnosticJStackService.java +++ /dev/null @@ -1,67 +0,0 @@ -/** * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.nodemanager; - -import org.apache.hadoop.yarn.server.nodemanager.webapp.DiagnosticJStackService; -import org.junit.jupiter.api.Test; - - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -public class TestDiagnosticJStackService { - - - - @Test - public void testExtractPidsFromEmptyProcessOutput(){ - String psOutput = ""; - - List pids = DiagnosticJStackService.extractPids(psOutput); - - assertTrue(pids.isEmpty()); - - } - - @Test - public void testExtractPidsFromOneProcessOutput(){ - String psOutput = "root 414 1.3 1.7 8124480 434520 ? Sl 11:36"; - - List pids = DiagnosticJStackService.extractPids(psOutput); - - assertEquals("414", pids.get(0)); - - } - - @Test - public void testExtractPidsFromMultipleProcessOutputs(){ - String psOutput = """ - root 414 1.3 1.7 8124480 434520 ? Sl 11:36 - root 420 1.3 1.7 8124480 434520 ? Sl 11:36 - """; - - List pids = DiagnosticJStackService.extractPids(psOutput); - - assertEquals("414", pids.get(0)); - assertEquals("420", pids.get(1)); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java new file mode 100644 index 00000000000000..b9c3f481199536 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -0,0 +1,33 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import org.junit.jupiter.api.Test; + + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class TestDiagnosticJStackService { + + + +} From 62f76d6128b440714505d00984830e8a914b9df6 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 26 Feb 2026 17:15:26 +0100 Subject: [PATCH 13/35] YARN-11823: Add unit test for the DiagnosticJStackService --- .../webapp/DiagnosticJStackService.java | 31 ++--- .../nodemanager/webapp/NMWebServices.java | 5 +- .../webapp/TestDiagnosticJStackService.java | 124 +++++++++++++++++- 3 files changed, 139 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 5dc292a456a0ac..4929df824ac730 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -44,17 +45,13 @@ public DiagnosticJStackService(Context context) { this.context = context; } - - public static String collectNodeThreadDump(int numberOfJStack) + public String collectNodeThreadDump(int numberOfJStack) throws IOException { - if (Shell.WINDOWS) { - throw new UnsupportedOperationException("Not implemented for Windows"); - } + checkShellNotWindows(); - long nodeManagerPid = getNodeManagerPid(); + long nodeManagerPid = ProcessHandle.current().pid(); return runJStack(nodeManagerPid, numberOfJStack); - } public String collectApplicationThreadDump(String appId, int numberOfJStack) @@ -62,22 +59,21 @@ public String collectApplicationThreadDump(String appId, int numberOfJStack) if(!appId.matches("application_\\d{13}_\\d{4}")) { throw new RuntimeException("Invalid application id: " + appId); } - - if (Shell.WINDOWS) { - throw new UnsupportedOperationException("Not implemented for Windows."); - } + checkShellNotWindows(); List applicationPids = getApplicationPids(appId); return runJStack(applicationPids, numberOfJStack); } - - public static long getNodeManagerPid() { - return ProcessHandle.current().pid(); + private void checkShellNotWindows() { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); + } } - public List getApplicationPids(String appId){ + @VisibleForTesting + List getApplicationPids(String appId){ List pids = new ArrayList<>(); ApplicationId appIdObj = ApplicationId.fromString(appId); @@ -102,8 +98,7 @@ public List getApplicationPids(String appId){ } - - public static String runJStack(List pids, int numJStacks) throws IOException { + private String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); for(Long pid : pids){ @@ -113,7 +108,7 @@ public static String runJStack(List pids, int numJStacks) throws IOExcepti return result.toString(); } - public static String runJStack(long pid, int numJStacks) throws IOException { + private String runJStack(long pid, int numJStacks) throws IOException { Optional processHandle = ProcessHandle.of(pid); if (processHandle.isEmpty()){ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index f4169920d0d9aa..1a519b45b4b377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -639,8 +639,9 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac { if (isJStackEndpointsEnable) { try { + DiagnosticJStackService diagnosticJStackService = new DiagnosticJStackService(this.nmContext); return Response.status(Status.OK) - .entity(DiagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack)) .build(); } catch (IOException e){ throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + @@ -682,7 +683,7 @@ public Response getApplicationJStack(@PathParam("appid") String appId, "For more information please check the NodeManager logs."); } } else { - return Response.status(Status.METHOD_NOT_ALLOWED) + return Response.status(Status.METHOD_NOT_ALLOWED) .build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index b9c3f481199536..161174096c2cd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -17,17 +17,139 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; - +import java.io.IOException; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.spy; public class TestDiagnosticJStackService { + private static final int NUMBER_OF_JSTACKS = 3; + private static final String DUMMY_JSTACK = + "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; + + private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( + null, null, null + , null, null, false, new Configuration() + ); + private static final DiagnosticJStackService diagnosticJStackService + = spy(new DiagnosticJStackService(nmContext)); + + + @Test + public void testWrongApplicationId() { + String applicationId = "app_29042"; + + assertThrows(RuntimeException.class, + () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3)); + } + + @Test + public void testCollectNodeThreadDump_Success() { + // No need to mock ProcessID, as it will take the unit test JVM PID + try(MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK) + ) // Wrap mockConstruction here to automatically close it + ){ + String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS); + + assertEquals(1, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated once only"); + + Shell.ShellCommandExecutor mockExecutor = mockedConstruction.constructed().get(0); + + // Verify the number of method calls + verify(mockExecutor, times(NUMBER_OF_JSTACKS)).execute(); + verify(mockExecutor, times(NUMBER_OF_JSTACKS)).getOutput(); + + assertTrue(result.contains("--- JStack iteration 0")); + assertTrue(result.contains("--- JStack iteration 1")); + assertTrue(result.contains("--- JStack iteration 2")); + assertTrue(result.contains(DUMMY_JSTACK)); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testCollectApplicationThreadDump_Success() { + String applicationId = "application_1771512066750_0001"; + + List pids = List.of(23L, 12L, 531L); + + doReturn(pids).when(diagnosticJStackService).getApplicationPids(applicationId); + + ProcessHandle mockProcessHandle = mock(ProcessHandle.class); + ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); + + when(mockProcessHandle.info()).thenReturn(mockPhInfo); + when(mockPhInfo.user()).thenReturn(Optional.empty()); + + try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); + MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + // Wrap mockedStatic & mockedConstruction here to automatically close them + ){ + mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); + + String result = diagnosticJStackService.collectApplicationThreadDump(applicationId, NUMBER_OF_JSTACKS); + + assertEquals(pids.size(), mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated for each PID"); + + Shell.ShellCommandExecutor mockExecutor1 = mockedConstruction.constructed().get(0); + // Verify the number of method calls + verify(mockExecutor1, times(NUMBER_OF_JSTACKS)).execute(); + verify(mockExecutor1, times(NUMBER_OF_JSTACKS)).getOutput(); + + assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); + assertTrue(result.contains(DUMMY_JSTACK)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + + @Test + public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { + String applicationId = "application_1771512066750_0001"; + int numJStacks = 3; + List pids = List.of(23L); + + doReturn(pids).when(diagnosticJStackService).getApplicationPids(applicationId); + + assertThrows(IOException.class, + () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, numJStacks), + "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); + + } + } From 3f2879638f6aea8d66f2e30a1e9751a00d1cbb80 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 27 Feb 2026 16:55:40 +0100 Subject: [PATCH 14/35] YARN-11823: fix checkstyle and bug issue --- .../webapp/DiagnosticJStackService.java | 140 +++++++++--------- 1 file changed, 68 insertions(+), 72 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 4929df824ac730..6429b4274a9c9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -37,103 +37,99 @@ public class DiagnosticJStackService { - private final Context context; - private static final Logger LOG = LoggerFactory - .getLogger(DiagnosticJStackService.class); + private final Context context; + private static final Logger LOG = LoggerFactory.getLogger(DiagnosticJStackService.class); - public DiagnosticJStackService(Context context) { - this.context = context; - } + public DiagnosticJStackService(Context context) { + this.context = context; + } + + public String collectNodeThreadDump(int numberOfJStack) throws IOException { + checkShellNotWindows(); - public String collectNodeThreadDump(int numberOfJStack) - throws IOException { - checkShellNotWindows(); + long nodeManagerPid = ProcessHandle.current().pid(); - long nodeManagerPid = ProcessHandle.current().pid(); + return runJStack(nodeManagerPid, numberOfJStack); + } - return runJStack(nodeManagerPid, numberOfJStack); + public String collectApplicationThreadDump(String appId, int numberOfJStack) throws IOException { + if(!appId.matches("application_\\d{13}_\\d{4}")) { + throw new RuntimeException("Invalid application id: " + appId); } + checkShellNotWindows(); - public String collectApplicationThreadDump(String appId, int numberOfJStack) - throws IOException { - if(!appId.matches("application_\\d{13}_\\d{4}")) { - throw new RuntimeException("Invalid application id: " + appId); - } - checkShellNotWindows(); + List applicationPids = getApplicationPids(appId); - List applicationPids = getApplicationPids(appId); + return runJStack(applicationPids, numberOfJStack); + } - return runJStack(applicationPids, numberOfJStack); + private void checkShellNotWindows() { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); } - - private void checkShellNotWindows() { - if (Shell.WINDOWS) { - throw new UnsupportedOperationException("Not implemented for Windows."); - } + } + + @VisibleForTesting + List getApplicationPids(String appId){ + List pids = new ArrayList<>(); + + ApplicationId appIdObj = ApplicationId.fromString(appId); + Application app = context.getApplications().get(appIdObj); + if (app != null) { + Map containers = app.getContainers(); + for (ContainerId containerId : containers.keySet()){ + long pidForContainerId = Long.parseLong(context.getContainerExecutor().getProcessId(containerId)); + + ProcessHandle.of(pidForContainerId).ifPresent(parentProcess -> + parentProcess.descendants() + .filter(childProcess -> + childProcess.info().command().orElse("").contains("java")) + .map(ProcessHandle::pid) + .forEach(pids::add) + ); + } } - @VisibleForTesting - List getApplicationPids(String appId){ - List pids = new ArrayList<>(); - - ApplicationId appIdObj = ApplicationId.fromString(appId); - Application app = context.getApplications().get(appIdObj); - if (app != null) { - Map containers = app.getContainers(); - for (ContainerId containerId : containers.keySet()){ - long pidForContainerId = Long.parseLong(context.getContainerExecutor().getProcessId(containerId)); - - ProcessHandle.of(pidForContainerId).ifPresent(parentProcess -> - parentProcess.descendants() - .filter(childProcess -> - childProcess.info().command().orElse("").contains("java")) - .map(ProcessHandle::pid) - .forEach(pids::add) - ); + LOG.info("Application PIDs: {}", pids); - } - } + return pids; + } - return pids; + private String runJStack(List pids, int numJStacks) throws IOException { + StringBuilder result = new StringBuilder(); + for(Long pid : pids){ + result.append(runJStack(pid, numJStacks)); } - private String runJStack(List pids, int numJStacks) throws IOException { - StringBuilder result = new StringBuilder(); + return result.toString(); + } - for(Long pid : pids){ - result.append(runJStack(pid, numJStacks)); - } + private String runJStack(long pid, int numJStacks) throws IOException { + Optional processHandle = ProcessHandle.of(pid); - return result.toString(); + if (processHandle.isEmpty()){ + throw new IOException("Process with PID " + pid + " is no longer exists"); } - private String runJStack(long pid, int numJStacks) throws IOException { - Optional processHandle = ProcessHandle.of(pid); + String processOwner = processHandle.get().info().user().orElse("root"); + String[] jstackCommand = {"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}; - if (processHandle.isEmpty()){ - throw new IOException("Process with PID " + pid + " is no longer exists"); - } + LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); - String processOwner = processHandle.get().info().user().orElse("root"); - String[] jstackCommand = {"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}; + Shell.ShellCommandExecutor cmd = + new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); + StringBuilder result = new StringBuilder(); - Shell.ShellCommandExecutor cmd = - new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - - StringBuilder result = new StringBuilder(); - - for (int i = 0; i < numJStacks; i++) { - cmd.execute(); - result.append(String.format( - "--- JStack iteration %d for PID: %d ---\n%s\n", i, pid, cmd.getOutput() - )); - } - - return result.toString(); + for (int i = 0; i < numJStacks; i++) { + cmd.execute(); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); } + return result.toString(); + } + } From cf2253b0d1335382c01c32ca1cd0e2a307c8afe7 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 2 Mar 2026 17:25:07 +0100 Subject: [PATCH 15/35] YARN-11823: clean up code at getApplicationPID to be more readable --- .../webapp/DiagnosticJStackService.java | 44 +++++++++---------- .../webapp/TestDiagnosticJStackService.java | 12 +++-- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 6429b4274a9c9f..cae39d8f177aac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -22,6 +22,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -34,11 +35,12 @@ import java.util.Map; import java.util.Optional; import java.util.Arrays; +import java.util.stream.Stream; public class DiagnosticJStackService { - private final Context context; private static final Logger LOG = LoggerFactory.getLogger(DiagnosticJStackService.class); + private final Context context; public DiagnosticJStackService(Context context) { this.context = context; @@ -53,12 +55,10 @@ public String collectNodeThreadDump(int numberOfJStack) throws IOException { } public String collectApplicationThreadDump(String appId, int numberOfJStack) throws IOException { - if(!appId.matches("application_\\d{13}_\\d{4}")) { - throw new RuntimeException("Invalid application id: " + appId); - } checkShellNotWindows(); - List applicationPids = getApplicationPids(appId); + ApplicationId applicationId = ApplicationId.fromString(appId); + List applicationPids = getApplicationPids(applicationId); return runJStack(applicationPids, numberOfJStack); } @@ -69,25 +69,25 @@ private void checkShellNotWindows() { } } - @VisibleForTesting - List getApplicationPids(String appId){ + protected List getApplicationPids(ApplicationId appId){ List pids = new ArrayList<>(); - ApplicationId appIdObj = ApplicationId.fromString(appId); - Application app = context.getApplications().get(appIdObj); - if (app != null) { - Map containers = app.getContainers(); - for (ContainerId containerId : containers.keySet()){ - long pidForContainerId = Long.parseLong(context.getContainerExecutor().getProcessId(containerId)); - - ProcessHandle.of(pidForContainerId).ifPresent(parentProcess -> - parentProcess.descendants() - .filter(childProcess -> - childProcess.info().command().orElse("").contains("java")) - .map(ProcessHandle::pid) - .forEach(pids::add) - ); - } + Application app = context.getApplications().get(appId); + if (app == null){ + throw new YarnRuntimeException("Application " + appId + " does not exist"); + } + + for (ContainerId containerId : app.getContainers().keySet()){ + String pidForContainerIdStr = context.getContainerExecutor().getProcessId(containerId); + long pidForContainerId = Long.parseLong(pidForContainerIdStr); + + ProcessHandle.of(pidForContainerId).ifPresent(handle -> + handle.descendants() // Get only the java processId of containerId's children + .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) + .map(ProcessHandle::pid) + .forEach(pids::add) + ); + } LOG.info("Application PIDs: {}", pids); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 161174096c2cd0..4f186cd293dbc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.junit.jupiter.api.Test; import org.mockito.MockedConstruction; @@ -95,7 +96,8 @@ public void testCollectNodeThreadDump_Success() { @Test public void testCollectApplicationThreadDump_Success() { - String applicationId = "application_1771512066750_0001"; + String applicationIdStr = "application_1771512066750_0001"; + ApplicationId applicationId = ApplicationId.fromString(applicationIdStr); List pids = List.of(23L, 12L, 531L); @@ -115,7 +117,7 @@ public void testCollectApplicationThreadDump_Success() { ){ mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - String result = diagnosticJStackService.collectApplicationThreadDump(applicationId, NUMBER_OF_JSTACKS); + String result = diagnosticJStackService.collectApplicationThreadDump(applicationIdStr, NUMBER_OF_JSTACKS); assertEquals(pids.size(), mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID"); @@ -138,14 +140,16 @@ public void testCollectApplicationThreadDump_Success() { @Test public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { - String applicationId = "application_1771512066750_0001"; + String applicationIdStr = "application_1771512066750_0001"; + ApplicationId applicationId = ApplicationId.fromString(applicationIdStr); + int numJStacks = 3; List pids = List.of(23L); doReturn(pids).when(diagnosticJStackService).getApplicationPids(applicationId); assertThrows(IOException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, numJStacks), + () -> diagnosticJStackService.collectApplicationThreadDump(applicationIdStr, numJStacks), "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); } From a57c52284bfb04c1c2e09bbe5d96a826fa863257 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Wed, 4 Mar 2026 17:48:31 +0100 Subject: [PATCH 16/35] YARN-11823: add a working solution without using sudo --- .../webapp/DiagnosticJStackService.java | 21 +++++++++++++++++-- .../impl/container-executor.c | 17 +++++++++++++++ .../impl/container-executor.h | 8 ++++++- .../native/container-executor/impl/main.c | 13 ++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index cae39d8f177aac..846723563c0ec9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -19,9 +19,12 @@ import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; @@ -29,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -112,8 +116,21 @@ private String runJStack(long pid, int numJStacks) throws IOException { throw new IOException("Process with PID " + pid + " is no longer exists"); } - String processOwner = processHandle.get().info().user().orElse("root"); - String[] jstackCommand = {"sudo", "-u", processOwner, "jstack", String.valueOf(pid)}; + String nmUser = System.getProperty("user.name"); + + String processOwner = processHandle.get().info().user().orElse(nmUser); + Configuration conf = context.getConf(); + + String yarnHomeEnvVar = System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key()); + File hadoopBin = new File(yarnHomeEnvVar, "bin"); + String defaultPath = new File(hadoopBin, "container-executor").getAbsolutePath(); + String containerExecutorPath = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, defaultPath); + + String javaHome = System.getProperty("java.home"); + String jstackPath = javaHome + "/bin/jstack"; + String[] jstackCommand = { + containerExecutorPath, "--run-jstack", processOwner, String.valueOf(pid), jstackPath + }; LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 57fd58494b14f6..1b5f7e6c1d9650 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -3310,3 +3310,20 @@ int remove_docker_container(char**argv, int argc) { } return exit_code; } + +int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path){ + struct passwd *user_info = get_user_info(user); // Initialise user info as required by set_user + + int exit_code = set_user(user); + if (exit_code != 0) { + fprintf(ERRORFILE, "Failed to set user to %s\n", user); + return exit_code; + } + + // Have the permission to run as another user + execlp(jstack_path, "jstack", pid, NULL); + + fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno)); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 8219a6755006eb..2b3e88c3d45a9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -54,7 +54,8 @@ enum operations { RUN_AS_USER_SYNC_YARN_SYSFS = 15, EXEC_CONTAINER = 16, RUN_RUNC_CONTAINER = 17, - REAP_RUNC_LAYER_MOUNTS = 18 + REAP_RUNC_LAYER_MOUNTS = 18, + RUN_JSTACK = 19 }; #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group" @@ -368,3 +369,8 @@ int is_terminal_support_enabled(); * Check if runC feature is enabled */ int is_runc_support_enabled(); + +/** +* Run jstack as process owner +*/ +int run_jstack_as_user(const char *user, const char *pid); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 1b91e8a3d6cd06..75e258c0e9cd7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -467,6 +467,16 @@ static int validate_arguments(int argc, char **argv , int *operation) { } } + if(strcmp("--run-jstack", argv[1]) == 0) { + if(argc != 4){ + fprintf(ERRORFILE, "Usage: container-executor --run-jstack \n"); + return INVALID_ARGUMENT_NUMBER; + } + + *operation = RUN_JSTACK; + return 0; + } + /* Now we have to validate 'run as user' operations that don't use a 'long option' - we should fix this at some point. The validation/argument @@ -825,6 +835,9 @@ int main(int argc, char **argv) { case REAP_RUNC_LAYER_MOUNTS: exit_code = reap_runc_layer_mounts(cmd_input.runc_layer_count); break; + case RUN_JSTACK: + exit_code = run_jstack_as_user(argv[2], argv[3]); + break; default: fprintf(ERRORFILE, "Unexpected operation code: %d\n", operation); exit_code = INVALID_COMMAND_PROVIDED; From 25ac6e12f3bd7a9f462dffb5e1777f46a88a4b05 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 5 Mar 2026 16:11:24 +0100 Subject: [PATCH 17/35] YARN-11823: fix the absolute path jstack issue --- .../native/container-executor/impl/container-executor.c | 4 ++-- .../native/container-executor/impl/container-executor.h | 4 ++-- .../src/main/native/container-executor/impl/main.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 1b5f7e6c1d9650..3a8e6774332477 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -3320,8 +3320,8 @@ int run_jstack_as_user(const char *user, const char *pid, const char *jstack_pat return exit_code; } - // Have the permission to run as another user - execlp(jstack_path, "jstack", pid, NULL); + //Use exec "$0" "$1" yo prevents buffer overflow vulnerabilities + execlp("/bin/bash", "bash", "-c", "exec \"$0\" \"$1\"", jstack_path, pid, NULL); fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno)); return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 2b3e88c3d45a9a..4327d939ac33c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -371,6 +371,6 @@ int is_terminal_support_enabled(); int is_runc_support_enabled(); /** -* Run jstack as process owner +* Run jstack as a specific user */ -int run_jstack_as_user(const char *user, const char *pid); +int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 75e258c0e9cd7c..6318b99e18bdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -468,8 +468,8 @@ static int validate_arguments(int argc, char **argv , int *operation) { } if(strcmp("--run-jstack", argv[1]) == 0) { - if(argc != 4){ - fprintf(ERRORFILE, "Usage: container-executor --run-jstack \n"); + if(argc != 5){ + fprintf(ERRORFILE, "Usage: container-executor --run-jstack \n"); return INVALID_ARGUMENT_NUMBER; } @@ -836,7 +836,7 @@ int main(int argc, char **argv) { exit_code = reap_runc_layer_mounts(cmd_input.runc_layer_count); break; case RUN_JSTACK: - exit_code = run_jstack_as_user(argv[2], argv[3]); + exit_code = run_jstack_as_user(argv[2], argv[3], argv[4]); break; default: fprintf(ERRORFILE, "Unexpected operation code: %d\n", operation); From 0dd487086565553f5f5e72cc65ce0cee1cc9252c Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 6 Mar 2026 16:21:07 +0100 Subject: [PATCH 18/35] YARN-11823: Clean up code to use ContainerExecutor effectively --- .../webapp/DiagnosticJStackService.java | 47 +++++++------------ .../nodemanager/webapp/NMWebServices.java | 1 + .../impl/container-executor.c | 5 +- .../webapp/TestDiagnosticJStackService.java | 27 ++++++----- 4 files changed, 34 insertions(+), 46 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 846723563c0ec9..6b4b3813e44f98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -18,36 +18,35 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; + import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Arrays; -import java.util.stream.Stream; public class DiagnosticJStackService { private static final Logger LOG = LoggerFactory.getLogger(DiagnosticJStackService.class); + + private static final String NM_USER = System.getProperty("user.name"); + private static final String JSTACK_PATH = System.getProperty("java.home") + "/bin/jstack"; private final Context context; + private final Configuration conf; public DiagnosticJStackService(Context context) { this.context = context; + this.conf = context.getConf(); } public String collectNodeThreadDump(int numberOfJStack) throws IOException { @@ -103,43 +102,32 @@ private String runJStack(List pids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); for(Long pid : pids){ - result.append(runJStack(pid, numJStacks)); + result.append(String.format( + "=== Thread Dumps for PID: %d ===%n%s%n", pid, runJStack(pid, numJStacks))); } return result.toString(); } private String runJStack(long pid, int numJStacks) throws IOException { - Optional processHandle = ProcessHandle.of(pid); - - if (processHandle.isEmpty()){ - throw new IOException("Process with PID " + pid + " is no longer exists"); - } + ProcessHandle processHandle = ProcessHandle.of(pid) + .orElseThrow(() -> new IOException("Process with PID " + pid + " is no longer exists")); - String nmUser = System.getProperty("user.name"); + String processOwner = processHandle.info().user().orElse(NM_USER); + String containerExecutorPath = PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); - String processOwner = processHandle.get().info().user().orElse(nmUser); - Configuration conf = context.getConf(); - - String yarnHomeEnvVar = System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key()); - File hadoopBin = new File(yarnHomeEnvVar, "bin"); - String defaultPath = new File(hadoopBin, "container-executor").getAbsolutePath(); - String containerExecutorPath = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, defaultPath); - - String javaHome = System.getProperty("java.home"); - String jstackPath = javaHome + "/bin/jstack"; String[] jstackCommand = { - containerExecutorPath, "--run-jstack", processOwner, String.valueOf(pid), jstackPath + containerExecutorPath, "--run-jstack", processOwner, String.valueOf(pid), JSTACK_PATH }; LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); - Shell.ShellCommandExecutor cmd = - new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - StringBuilder result = new StringBuilder(); for (int i = 0; i < numJStacks; i++) { + Shell.ShellCommandExecutor cmd = + new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); + cmd.execute(); result.append(String.format( "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); @@ -148,5 +136,4 @@ private String runJStack(long pid, int numJStacks) throws IOException { return result.toString(); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index d1ffbeb426e889..b8a071dff76515 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfoes; import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 3a8e6774332477..ee7f72ef2e58d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -3312,15 +3312,14 @@ int remove_docker_container(char**argv, int argc) { } int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path){ - struct passwd *user_info = get_user_info(user); // Initialise user info as required by set_user - int exit_code = set_user(user); if (exit_code != 0) { fprintf(ERRORFILE, "Failed to set user to %s\n", user); return exit_code; } - //Use exec "$0" "$1" yo prevents buffer overflow vulnerabilities + // Use exec "$0" "$1" to treat as positional arguments, + // preventing command injection vulnerability CVE-2023-25555 execlp("/bin/bash", "bash", "-c", "exec \"$0\" \"$1\"", jstack_path, pid, NULL); fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 4f186cd293dbc1..fd05e9336a7c63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -75,14 +75,14 @@ public void testCollectNodeThreadDump_Success() { ){ String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS); - assertEquals(1, mockedConstruction.constructed().size(), - "ShellCommandExecutor should be instantiated once only"); + assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated relative to Number of JStacks"); - Shell.ShellCommandExecutor mockExecutor = mockedConstruction.constructed().get(0); - - // Verify the number of method calls - verify(mockExecutor, times(NUMBER_OF_JSTACKS)).execute(); - verify(mockExecutor, times(NUMBER_OF_JSTACKS)).getOutput(); + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } assertTrue(result.contains("--- JStack iteration 0")); assertTrue(result.contains("--- JStack iteration 1")); @@ -119,13 +119,14 @@ public void testCollectApplicationThreadDump_Success() { String result = diagnosticJStackService.collectApplicationThreadDump(applicationIdStr, NUMBER_OF_JSTACKS); - assertEquals(pids.size(), mockedConstruction.constructed().size(), - "ShellCommandExecutor should be instantiated for each PID"); + assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); - Shell.ShellCommandExecutor mockExecutor1 = mockedConstruction.constructed().get(0); - // Verify the number of method calls - verify(mockExecutor1, times(NUMBER_OF_JSTACKS)).execute(); - verify(mockExecutor1, times(NUMBER_OF_JSTACKS)).getOutput(); + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); From 75bcbab69e17c1bacdee161577f5e7eb86a1b0ff Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 9 Mar 2026 14:24:49 +0100 Subject: [PATCH 19/35] YARN-11823: cleaned up code and to display containerId along with processId --- .../webapp/DiagnosticJStackService.java | 51 ++++++++------ .../nodemanager/webapp/NMWebServices.java | 67 +++++++++---------- .../webapp/TestDiagnosticJStackService.java | 28 +++++--- 3 files changed, 80 insertions(+), 66 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 6b4b3813e44f98..1ff88bcc157bce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -31,9 +31,10 @@ import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; public class DiagnosticJStackService { @@ -61,9 +62,9 @@ public String collectApplicationThreadDump(String appId, int numberOfJStack) thr checkShellNotWindows(); ApplicationId applicationId = ApplicationId.fromString(appId); - List applicationPids = getApplicationPids(applicationId); + Map> containerPids = getApplicationContainerPids(applicationId); - return runJStack(applicationPids, numberOfJStack); + return runJStack(containerPids, numberOfJStack); } private void checkShellNotWindows() { @@ -72,38 +73,46 @@ private void checkShellNotWindows() { } } - protected List getApplicationPids(ApplicationId appId){ - List pids = new ArrayList<>(); - + protected Map> getApplicationContainerPids(ApplicationId appId){ Application app = context.getApplications().get(appId); if (app == null){ throw new YarnRuntimeException("Application " + appId + " does not exist"); } + Map> containerPids = new HashMap<>(); + for (ContainerId containerId : app.getContainers().keySet()){ String pidForContainerIdStr = context.getContainerExecutor().getProcessId(containerId); - long pidForContainerId = Long.parseLong(pidForContainerIdStr); + long parentPid = Long.parseLong(pidForContainerIdStr); - ProcessHandle.of(pidForContainerId).ifPresent(handle -> - handle.descendants() // Get only the java processId of containerId's children - .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) - .map(ProcessHandle::pid) - .forEach(pids::add) - ); + List javaContainerPids = ProcessHandle.of(parentPid).stream() + .flatMap(ProcessHandle::descendants) + .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) + .map(ProcessHandle::pid) + .toList(); + if (!javaContainerPids.isEmpty()){ + containerPids.put(containerId, javaContainerPids); + } } - LOG.info("Application PIDs: {}", pids); + LOG.info("Application PIDs by ContainerId: {}", containerPids); - return pids; + return containerPids; } - private String runJStack(List pids, int numJStacks) throws IOException { + private String runJStack(Map> containerPids, int numJStacks) throws IOException { StringBuilder result = new StringBuilder(); - for(Long pid : pids){ - result.append(String.format( - "=== Thread Dumps for PID: %d ===%n%s%n", pid, runJStack(pid, numJStacks))); + for(ContainerId containerPid : containerPids.keySet()){ + List javaContainerPids = containerPids.get(containerPid); + + for (Long pid : javaContainerPids){ + result.append(String.format( + "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", + containerPid.toString(), pid, runJStack(pid, numJStacks))); + } + } return result.toString(); @@ -113,11 +122,11 @@ private String runJStack(long pid, int numJStacks) throws IOException { ProcessHandle processHandle = ProcessHandle.of(pid) .orElseThrow(() -> new IOException("Process with PID " + pid + " is no longer exists")); - String processOwner = processHandle.info().user().orElse(NM_USER); + String runningUser = processHandle.info().user().orElse(NM_USER); String containerExecutorPath = PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); String[] jstackCommand = { - containerExecutorPath, "--run-jstack", processOwner, String.valueOf(pid), JSTACK_PATH + containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH }; LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index b8a071dff76515..5bfaa372063fb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -108,6 +108,7 @@ public class NMWebServices { private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private String redirectWSUrl; private LogAggregationFileControllerFactory factory; + private DiagnosticJStackService diagnosticJStackService; private boolean filterAppsByUser = false; private boolean isJStackEndpointsEnable = false; @@ -138,6 +139,7 @@ public NMWebServices(final @javax.inject.Named("nm") Context nm, YarnConfiguration.NM_JSTACK_ENDPOINTS_ENABLED, YarnConfiguration.DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED ); + this.diagnosticJStackService = new DiagnosticJStackService(this.nmContext); } public NMWebServices(final Context nm, final ResourceView view, @@ -639,24 +641,23 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context @Path("/jstack/{numberOfJStack}") public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) { - if (isJStackEndpointsEnable) { - try { - DiagnosticJStackService diagnosticJStackService = new DiagnosticJStackService(this.nmContext); - return Response.status(Status.OK) - .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack)) - .build(); - } catch (IOException e){ - throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } catch (Exception e) { - throw new WebAppException( - "Unexpected error collection NodeManager JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } - } else { + if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) .build(); } + + try { + return Response.status(Status.OK) + .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + .build(); + } catch (IOException e){ + throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } catch (Exception e) { + throw new WebAppException( + "Unexpected error collection NodeManager JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } } @@ -666,29 +667,27 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac public Response getApplicationJStack(@PathParam("appid") String appId, @PathParam("numberOfJStack") int numberOfJStack) { - if (isJStackEndpointsEnable) { - try { - DiagnosticJStackService diagnosticJStackService = new DiagnosticJStackService(this.nmContext); - - return Response.status(Status.OK) - .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) - .build(); - } catch (RuntimeException e){ - throw new WebAppException( - "The applicationId is invalid: " + appId + "." + e.getMessage()); - } catch (IOException e){ - throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } catch (Exception e) { - throw new WebAppException( - "Unexpected error collecting Application JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); - } - } else { + if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) .build(); } + try { + return Response.status(Status.OK) + .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) + .build(); + } catch (RuntimeException e){ + throw new WebAppException( + "The applicationId is invalid: " + appId + "." + e.getMessage()); + } catch (IOException e){ + throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } catch (Exception e) { + throw new WebAppException( + "Unexpected error collecting Application JStack: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs."); + } + } private long parseLongParam(String bytes) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index fd05e9336a7c63..db59ecc2860c13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -20,13 +20,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.junit.jupiter.api.Test; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -48,6 +51,11 @@ public class TestDiagnosticJStackService { private static final int NUMBER_OF_JSTACKS = 3; private static final String DUMMY_JSTACK = "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; + private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; + private static final ApplicationId APPLICATION_ID = + ApplicationId.fromString(APPLICATION_ID_STR); + private static final ContainerId CONTAINER_ID = + ContainerId.fromString("container_1771512066750_0001_01_000049"); private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( null, null, null @@ -96,12 +104,11 @@ public void testCollectNodeThreadDump_Success() { @Test public void testCollectApplicationThreadDump_Success() { - String applicationIdStr = "application_1771512066750_0001"; - ApplicationId applicationId = ApplicationId.fromString(applicationIdStr); - List pids = List.of(23L, 12L, 531L); - doReturn(pids).when(diagnosticJStackService).getApplicationPids(applicationId); + Map> containerPids = Map.of(CONTAINER_ID, pids); + + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(APPLICATION_ID); ProcessHandle mockProcessHandle = mock(ProcessHandle.class); ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); @@ -117,7 +124,7 @@ public void testCollectApplicationThreadDump_Success() { ){ mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - String result = diagnosticJStackService.collectApplicationThreadDump(applicationIdStr, NUMBER_OF_JSTACKS); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS); assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); @@ -141,16 +148,15 @@ public void testCollectApplicationThreadDump_Success() { @Test public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { - String applicationIdStr = "application_1771512066750_0001"; - ApplicationId applicationId = ApplicationId.fromString(applicationIdStr); - int numJStacks = 3; - List pids = List.of(23L); + List pids = List.of(23L, 12L, 531L); + + Map> containerPids = Map.of(CONTAINER_ID, pids); - doReturn(pids).when(diagnosticJStackService).getApplicationPids(applicationId); + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(APPLICATION_ID); assertThrows(IOException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationIdStr, numJStacks), + () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks), "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); } From a455836fce199499e0e6f9a86b25d2c1eb7e477f Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 10 Mar 2026 17:23:49 +0100 Subject: [PATCH 20/35] YARN-11823: remove catching general exception exceptions and add text plain reproduce --- .../server/nodemanager/webapp/NMWebServices.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 5bfaa372063fb1..16adc14a2742c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -639,6 +639,7 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context @GET @Path("/jstack/{numberOfJStack}") + @Produces({ MediaType.TEXT_PLAIN}) public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) { if (!isJStackEndpointsEnable) { @@ -653,11 +654,8 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac } catch (IOException e){ throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + "For more information please check the NodeManager logs."); - } catch (Exception e) { - throw new WebAppException( - "Unexpected error collection NodeManager JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); } + } @@ -676,16 +674,12 @@ public Response getApplicationJStack(@PathParam("appid") String appId, return Response.status(Status.OK) .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) .build(); - } catch (RuntimeException e){ + } catch (IllegalArgumentException e){ throw new WebAppException( "The applicationId is invalid: " + appId + "." + e.getMessage()); } catch (IOException e){ throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + "For more information please check the NodeManager logs."); - } catch (Exception e) { - throw new WebAppException( - "Unexpected error collecting Application JStack: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); } } From 3431c954327b0d8bfaeef16f26ed4082a7dcfa6e Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 13 Mar 2026 16:26:38 +0100 Subject: [PATCH 21/35] YARN-11823: check application authorization and put the non java containerId in the response --- .../webapp/DiagnosticJStackService.java | 65 ++++++++++++++----- .../nodemanager/webapp/NMWebServices.java | 7 +- .../webapp/TestDiagnosticJStackService.java | 16 +++-- 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 1ff88bcc157bce..90fc67f7deb418 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -19,17 +19,21 @@ import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.List; import java.util.Arrays; @@ -58,27 +62,53 @@ public String collectNodeThreadDump(int numberOfJStack) throws IOException { return runJStack(nodeManagerPid, numberOfJStack); } - public String collectApplicationThreadDump(String appId, int numberOfJStack) throws IOException { + public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) throws IOException { checkShellNotWindows(); ApplicationId applicationId = ApplicationId.fromString(appId); - Map> containerPids = getApplicationContainerPids(applicationId); + + Application app = context.getApplications().get(applicationId); + if (app == null){ + throw new YarnRuntimeException("Application " + applicationId + " does not exist"); + } + + checkApplicationACL(req, app); + + Map> containerPids = getApplicationContainerPids(app); return runJStack(containerPids, numberOfJStack); } + private void checkApplicationACL(HttpServletRequest req, Application app) throws IOException { + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI; + + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } else { + callerUGI = UserGroupInformation.getCurrentUser(); // Fallback to the current OS User + } + + LOG.info("Caller UGI: {}", callerUGI.toString()); + + boolean isAuthorized = context.getApplicationACLsManager().checkAccess( + callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId() + ); + + if(!isAuthorized){ + throw new WebAppException("User " + callerUGI.getShortUserName() + + " is not authorized to view application " + app.getAppId()); + } + + } + private void checkShellNotWindows() { if (Shell.WINDOWS) { throw new UnsupportedOperationException("Not implemented for Windows."); } } - protected Map> getApplicationContainerPids(ApplicationId appId){ - Application app = context.getApplications().get(appId); - if (app == null){ - throw new YarnRuntimeException("Application " + appId + " does not exist"); - } - + protected Map> getApplicationContainerPids(Application app){ Map> containerPids = new HashMap<>(); for (ContainerId containerId : app.getContainers().keySet()){ @@ -91,9 +121,9 @@ protected Map> getApplicationContainerPids(ApplicationId .map(ProcessHandle::pid) .toList(); - if (!javaContainerPids.isEmpty()){ - containerPids.put(containerId, javaContainerPids); - } + + containerPids.put(containerId, javaContainerPids); + } LOG.info("Application PIDs by ContainerId: {}", containerPids); @@ -107,10 +137,15 @@ private String runJStack(Map> containerPids, int numJSta for(ContainerId containerPid : containerPids.keySet()){ List javaContainerPids = containerPids.get(containerPid); - for (Long pid : javaContainerPids){ - result.append(String.format( - "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", - containerPid.toString(), pid, runJStack(pid, numJStacks))); + if (javaContainerPids.isEmpty()){ + result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " + + "because no Java Process ID exist ===", containerPid.toString())); + } else { + for (Long pid : javaContainerPids) { + result.append(String.format( + "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", + containerPid.toString(), pid, runJStack(pid, numJStacks))); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 16adc14a2742c2..65ddf273a9bbe7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -662,8 +662,9 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac @GET @Path("/apps/{appid}/jstack/{numberOfJStack}") @Produces({MediaType.TEXT_PLAIN}) - public Response getApplicationJStack(@PathParam("appid") String appId, - @PathParam("numberOfJStack") int numberOfJStack) + public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletRequest req, + @PathParam("appid") String appId, + @PathParam("numberOfJStack") int numberOfJStack) { if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) @@ -672,7 +673,7 @@ public Response getApplicationJStack(@PathParam("appid") String appId, try { return Response.status(Status.OK) - .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack)) + .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack, req)) .build(); } catch (IllegalArgumentException e){ throw new WebAppException( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index db59ecc2860c13..d6e218b5e6afa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -22,7 +22,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.junit.jupiter.api.Test; +import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; @@ -70,7 +72,7 @@ public void testWrongApplicationId() { String applicationId = "app_29042"; assertThrows(RuntimeException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3)); + () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); } @Test @@ -105,10 +107,12 @@ public void testCollectNodeThreadDump_Success() { @Test public void testCollectApplicationThreadDump_Success() { List pids = List.of(23L, 12L, 531L); + Application app = mock(Application.class); + when(nmContext.getApplications().get(APPLICATION_ID)).thenReturn(app); Map> containerPids = Map.of(CONTAINER_ID, pids); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(APPLICATION_ID); + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(app); ProcessHandle mockProcessHandle = mock(ProcessHandle.class); ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); @@ -124,7 +128,7 @@ public void testCollectApplicationThreadDump_Success() { ){ mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, null); assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); @@ -150,13 +154,15 @@ public void testCollectApplicationThreadDump_Success() { public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { int numJStacks = 3; List pids = List.of(23L, 12L, 531L); + Application app = mock(Application.class); + when(nmContext.getApplications().get(APPLICATION_ID)).thenReturn(app); Map> containerPids = Map.of(CONTAINER_ID, pids); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(APPLICATION_ID); + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(app); assertThrows(IOException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks), + () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, null), "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); } From dd0c140cfbe9f377b43b48a13947fcfabd7cf4e7 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 17 Mar 2026 13:16:45 +0100 Subject: [PATCH 22/35] YARN-11823: fix handling the exception --- .../server/nodemanager/webapp/DiagnosticJStackService.java | 2 +- .../hadoop/yarn/server/nodemanager/webapp/NMWebServices.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 90fc67f7deb418..501a3d5032ebdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -96,7 +96,7 @@ private void checkApplicationACL(HttpServletRequest req, Application app) throws ); if(!isAuthorized){ - throw new WebAppException("User " + callerUGI.getShortUserName() + + throw new YarnRuntimeException("User " + callerUGI.getShortUserName() + " is not authorized to view application " + app.getAppId()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 65ddf273a9bbe7..0dbb9a75cedaf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; @@ -678,6 +679,8 @@ public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletReques } catch (IllegalArgumentException e){ throw new WebAppException( "The applicationId is invalid: " + appId + "." + e.getMessage()); + } catch (YarnRuntimeException e) { + throw new WebAppException(e.getMessage()); } catch (IOException e){ throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + "For more information please check the NodeManager logs."); From fdff84577f71ecb859c68b4567546887cb6601c2 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 17 Mar 2026 14:04:56 +0100 Subject: [PATCH 23/35] YARN-11823: fix the exception response --- .../nodemanager/webapp/NMWebServices.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 0dbb9a75cedaf3..6af2b6c50b72c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -653,8 +653,10 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack)) .build(); } catch (IOException e){ - throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs.") + .build(); } } @@ -677,13 +679,18 @@ public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletReques .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack, req)) .build(); } catch (IllegalArgumentException e){ - throw new WebAppException( - "The applicationId is invalid: " + appId + "." + e.getMessage()); + return Response.status(Status.BAD_REQUEST) + .entity("The applicationId is invalid: " + appId + ". " + e.getMessage()) + .build(); } catch (YarnRuntimeException e) { - throw new WebAppException(e.getMessage()); + return Response.status(Status.FORBIDDEN) + .entity(e.getMessage()) + .build(); } catch (IOException e){ - throw new WebAppException("Shell command has failed: " + e.getMessage() + ". " + - "For more information please check the NodeManager logs."); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs.") + .build(); } } From 5168e01ecf3f9078395935131fe618b1c38fac83 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 20 Mar 2026 17:41:13 +0100 Subject: [PATCH 24/35] YARN-11823: Use commandLine to get the java process and handle exception when running jstack for specific pid in case it exited during running --- .../webapp/DiagnosticJStackService.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 501a3d5032ebdf..3ee7e516cfabc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; -import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,7 +116,7 @@ protected Map> getApplicationContainerPids(Application a List javaContainerPids = ProcessHandle.of(parentPid).stream() .flatMap(ProcessHandle::descendants) - .filter(childProcess -> childProcess.info().command().orElse("").contains("java")) + .filter(childProcess -> childProcess.info().commandLine().orElse("").contains("java")) .map(ProcessHandle::pid) .toList(); @@ -172,9 +171,25 @@ private String runJStack(long pid, int numJStacks) throws IOException { Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - cmd.execute(); - result.append(String.format( - "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + try { + cmd.execute(); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + } catch (org.apache.hadoop.util.Shell.ExitCodeException e) { + LOG.warn("Failed to jstack PID {} (Process likely exited): {}", pid, e.getMessage()); + result.append(String.format("--- JStack iteration %d for PID: %d ---%n", i, pid)); + result.append("Status: Failed to collect thread dump. The process likely exited naturally before jstack could attach.\n"); + result.append("Error details: ").append(e.getMessage()).append("\n"); + + break; + } catch (IOException e) { + LOG.warn("IO Error while running jstack for PID {}: {}", pid, e.getMessage()); + result.append(String.format("--- JStack iteration %d for PID: %d ---%n", i, pid)); + result.append("Status: Incomplete read (Premature EOF). The JVM likely shut down while writing the thread dump.\n"); + esult.append("Error details: ").append(e.getMessage()).append("\n"); + + break; + } } return result.toString(); From 563278a8230ea4f48f364db48f46651393c52b2b Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 23 Mar 2026 16:57:41 +0100 Subject: [PATCH 25/35] YARN-11823: fix unit-test after adding security measure --- .../webapp/DiagnosticJStackService.java | 2 +- .../webapp/TestDiagnosticJStackService.java | 49 ++++++++++++------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 3ee7e516cfabc0..2d329ed5b10298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -186,7 +186,7 @@ private String runJStack(long pid, int numJStacks) throws IOException { LOG.warn("IO Error while running jstack for PID {}: {}", pid, e.getMessage()); result.append(String.format("--- JStack iteration %d for PID: %d ---%n", i, pid)); result.append("Status: Incomplete read (Premature EOF). The JVM likely shut down while writing the thread dump.\n"); - esult.append("Error details: ").append(e.getMessage()).append("\n"); + result.append("Error details: ").append(e.getMessage()).append("\n"); break; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index d6e218b5e6afa8..1666e41175d776 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -21,22 +21,25 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentMap; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -56,12 +59,14 @@ public class TestDiagnosticJStackService { private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; private static final ApplicationId APPLICATION_ID = ApplicationId.fromString(APPLICATION_ID_STR); + private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; private static final ContainerId CONTAINER_ID = - ContainerId.fromString("container_1771512066750_0001_01_000049"); + ContainerId.fromString(CONTAINER_ID_STR); + private static final ApplicationACLsManager mockAclManager = mock(ApplicationACLsManager.class); private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( null, null, null - , null, null, false, new Configuration() + , mockAclManager, null, false, new Configuration() ); private static final DiagnosticJStackService diagnosticJStackService = spy(new DiagnosticJStackService(nmContext)); @@ -107,12 +112,14 @@ public void testCollectNodeThreadDump_Success() { @Test public void testCollectApplicationThreadDump_Success() { List pids = List.of(23L, 12L, 531L); - Application app = mock(Application.class); - when(nmContext.getApplications().get(APPLICATION_ID)).thenReturn(app); - Map> containerPids = Map.of(CONTAINER_ID, pids); + Application mockApp = mock(Application.class); + nmContext.getApplications().put(APPLICATION_ID, mockApp); + + when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(app); + Map> containerPids = Map.of(CONTAINER_ID, pids); + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); ProcessHandle mockProcessHandle = mock(ProcessHandle.class); ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); @@ -128,7 +135,8 @@ public void testCollectApplicationThreadDump_Success() { ){ mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, null); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); @@ -147,23 +155,30 @@ public void testCollectApplicationThreadDump_Success() { throw new RuntimeException(e); } + nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + } @Test public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { int numJStacks = 3; - List pids = List.of(23L, 12L, 531L); - Application app = mock(Application.class); - when(nmContext.getApplications().get(APPLICATION_ID)).thenReturn(app); + Application mockApp = mock(Application.class); + nmContext.getApplications().put(APPLICATION_ID, mockApp); - Map> containerPids = Map.of(CONTAINER_ID, pids); + when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); + + Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(app); + doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); + + HttpServletRequest mockRequest = mock(HttpServletRequest.class); assertThrows(IOException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, null), - "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); + () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest), + "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); + + nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test } From 9f76e7577abb2a207b1d9975d21c60b6c48669e2 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 23 Mar 2026 18:03:56 +0100 Subject: [PATCH 26/35] YARN-11823: Handle the PID no longer exist without exception to not break the whole response --- .../webapp/DiagnosticJStackService.java | 39 +++++++------------ .../webapp/TestDiagnosticJStackService.java | 10 +++-- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 2d329ed5b10298..ddeae81fbcf2ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -34,10 +34,7 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.List; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; public class DiagnosticJStackService { @@ -153,8 +150,15 @@ private String runJStack(Map> containerPids, int numJSta } private String runJStack(long pid, int numJStacks) throws IOException { - ProcessHandle processHandle = ProcessHandle.of(pid) - .orElseThrow(() -> new IOException("Process with PID " + pid + " is no longer exists")); + Optional processHandleOpt = ProcessHandle.of(pid); + + if (processHandleOpt.isEmpty()){ + String msg = String.format("Process with PID " + pid + " is no longer exists"); + LOG.warn(msg); + return "Status: Skipped Process with PID " + msg; + } + + ProcessHandle processHandle = processHandleOpt.get(); String runningUser = processHandle.info().user().orElse(NM_USER); String containerExecutorPath = PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); @@ -171,25 +175,10 @@ private String runJStack(long pid, int numJStacks) throws IOException { Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - try { - cmd.execute(); - result.append(String.format( - "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); - } catch (org.apache.hadoop.util.Shell.ExitCodeException e) { - LOG.warn("Failed to jstack PID {} (Process likely exited): {}", pid, e.getMessage()); - result.append(String.format("--- JStack iteration %d for PID: %d ---%n", i, pid)); - result.append("Status: Failed to collect thread dump. The process likely exited naturally before jstack could attach.\n"); - result.append("Error details: ").append(e.getMessage()).append("\n"); - - break; - } catch (IOException e) { - LOG.warn("IO Error while running jstack for PID {}: {}", pid, e.getMessage()); - result.append(String.format("--- JStack iteration %d for PID: %d ---%n", i, pid)); - result.append("Status: Incomplete read (Premature EOF). The JVM likely shut down while writing the thread dump.\n"); - result.append("Error details: ").append(e.getMessage()).append("\n"); - - break; - } + cmd.execute(); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + } return result.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 1666e41175d776..d4430546c424b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -161,7 +161,7 @@ public void testCollectApplicationThreadDump_Success() { @Test - public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { + public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { int numJStacks = 3; Application mockApp = mock(Application.class); nmContext.getApplications().put(APPLICATION_ID, mockApp); @@ -174,8 +174,12 @@ public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() { HttpServletRequest mockRequest = mock(HttpServletRequest.class); - assertThrows(IOException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest), + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + + assertNotNull(result); + assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), + "Output should contain the container ID"); + assertTrue(result.contains("Status: Skipped Process with PID"), "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test From cde856a45ead2415ce5f5616dc22bde4e72dac00 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 23 Mar 2026 18:30:26 +0100 Subject: [PATCH 27/35] YARN-11823: Add a check for AdminACL to run NodeManager Jstack --- .../webapp/DiagnosticJStackService.java | 49 ++++++++++++++----- .../nodemanager/webapp/NMWebServices.java | 9 +++- .../webapp/TestDiagnosticJStackService.java | 5 +- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index ddeae81fbcf2ec..9b94df8efbf47c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -34,7 +34,11 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.*; +import java.util.Map; +import java.util.List; +import java.util.HashMap; +import java.util.Optional; +import java.util.Arrays; public class DiagnosticJStackService { @@ -50,14 +54,27 @@ public DiagnosticJStackService(Context context) { this.conf = context.getConf(); } - public String collectNodeThreadDump(int numberOfJStack) throws IOException { + public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req) throws IOException { checkShellNotWindows(); long nodeManagerPid = ProcessHandle.current().pid(); + checkAdminACL(req); + return runJStack(nodeManagerPid, numberOfJStack); } + private void checkAdminACL(HttpServletRequest req) throws IOException { + UserGroupInformation callerUGI = getUserGroupInformation(req); + + boolean isAdmin = context.getApplicationACLsManager().isAdmin(callerUGI); + + if (!isAdmin) { + throw new YarnRuntimeException("User " + callerUGI.getShortUserName() + + " is not authorized to run jstack on NodeManager "); + } + } + public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) throws IOException { checkShellNotWindows(); @@ -76,16 +93,7 @@ public String collectApplicationThreadDump(String appId, int numberOfJStack, Htt } private void checkApplicationACL(HttpServletRequest req, Application app) throws IOException { - String remoteUser = req.getRemoteUser(); - UserGroupInformation callerUGI; - - if (remoteUser != null) { - callerUGI = UserGroupInformation.createRemoteUser(remoteUser); - } else { - callerUGI = UserGroupInformation.getCurrentUser(); // Fallback to the current OS User - } - - LOG.info("Caller UGI: {}", callerUGI.toString()); + UserGroupInformation callerUGI = getUserGroupInformation(req); boolean isAuthorized = context.getApplicationACLsManager().checkAccess( callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId() @@ -184,4 +192,21 @@ private String runJStack(long pid, int numJStacks) throws IOException { return result.toString(); } + private UserGroupInformation getUserGroupInformation(HttpServletRequest req) throws IOException { + + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI; + + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } else { + callerUGI = UserGroupInformation.getCurrentUser(); // Fallback to current OS user + } + + LOG.info("Checking ACL for Caller UGI: {}", callerUGI.toString()); + + return callerUGI; + + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 6af2b6c50b72c7..c54e3c9090203f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -641,7 +641,8 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context @GET @Path("/jstack/{numberOfJStack}") @Produces({ MediaType.TEXT_PLAIN}) - public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStack) + public Response getNodeThreadDump(@javax.ws.rs.core.Context HttpServletRequest req, + @PathParam("numberOfJStack") int numberOfJStack) { if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) @@ -650,7 +651,11 @@ public Response getNodeThreadDump(@PathParam("numberOfJStack") int numberOfJStac try { return Response.status(Status.OK) - .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack)) + .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack, req)) + .build(); + } catch (YarnRuntimeException e) { + return Response.status(Status.FORBIDDEN) + .entity(e.getMessage()) .build(); } catch (IOException e){ return Response.status(Status.INTERNAL_SERVER_ERROR) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index d4430546c424b1..5b0d5076103f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -88,7 +88,10 @@ public void testCollectNodeThreadDump_Success() { (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK) ) // Wrap mockConstruction here to automatically close it ){ - String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + when(mockAclManager.isAdmin(any())).thenReturn(true); + + String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated relative to Number of JStacks"); From 86abf94d3c3e1c157576e1542584144c30193efa Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 27 Mar 2026 16:11:04 +0100 Subject: [PATCH 28/35] YARN-11823: Update to match exactly java process and handle exception when pid is exited while running jstack --- .../webapp/DiagnosticJStackService.java | 43 +++- .../webapp/TestDiagnosticJStackService.java | 210 +++++++++--------- 2 files changed, 134 insertions(+), 119 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 9b94df8efbf47c..930604b312e162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -54,7 +54,9 @@ public DiagnosticJStackService(Context context) { this.conf = context.getConf(); } - public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req) throws IOException { + public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req) + throws IOException + { checkShellNotWindows(); long nodeManagerPid = ProcessHandle.current().pid(); @@ -75,7 +77,9 @@ private void checkAdminACL(HttpServletRequest req) throws IOException { } } - public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) throws IOException { + public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) + throws IOException + { checkShellNotWindows(); ApplicationId applicationId = ApplicationId.fromString(appId); @@ -121,11 +125,20 @@ protected Map> getApplicationContainerPids(Application a List javaContainerPids = ProcessHandle.of(parentPid).stream() .flatMap(ProcessHandle::descendants) - .filter(childProcess -> childProcess.info().commandLine().orElse("").contains("java")) + .filter(childProcess -> { + String cmdLine = childProcess.info().commandLine().orElse("").trim(); + // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk-cloudera/bin/java + // -Djava.net.preferIPv4Stack=true + if (cmdLine.isEmpty()){ + return false; + } + + String executable = cmdLine.split("\\s+")[0]; // The first token is always the executable binary + return executable.equals("java") || executable.endsWith("/java"); + }) .map(ProcessHandle::pid) .toList(); - containerPids.put(containerId, javaContainerPids); } @@ -135,7 +148,7 @@ protected Map> getApplicationContainerPids(Application a return containerPids; } - private String runJStack(Map> containerPids, int numJStacks) throws IOException { + private String runJStack(Map> containerPids, int numJStacks){ StringBuilder result = new StringBuilder(); for(ContainerId containerPid : containerPids.keySet()){ @@ -157,7 +170,7 @@ private String runJStack(Map> containerPids, int numJSta return result.toString(); } - private String runJStack(long pid, int numJStacks) throws IOException { + private String runJStack(long pid, int numJStacks) { Optional processHandleOpt = ProcessHandle.of(pid); if (processHandleOpt.isEmpty()){ @@ -169,7 +182,8 @@ private String runJStack(long pid, int numJStacks) throws IOException { ProcessHandle processHandle = processHandleOpt.get(); String runningUser = processHandle.info().user().orElse(NM_USER); - String containerExecutorPath = PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); + String containerExecutorPath = + PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); String[] jstackCommand = { containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH @@ -183,17 +197,22 @@ private String runJStack(long pid, int numJStacks) throws IOException { Shell.ShellCommandExecutor cmd = new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); - cmd.execute(); - result.append(String.format( - "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); - + try { + cmd.execute(); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + } catch (IOException e) { + result.append(String.format( + "Failed to run jstack on PID: " + pid + " at iteration: " + i + + " (Process likely exited before/during running jstack): " + e.getMessage())); + break; + } } return result.toString(); } private UserGroupInformation getUserGroupInformation(HttpServletRequest req) throws IOException { - String remoteUser = req.getRemoteUser(); UserGroupInformation callerUGI; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 5b0d5076103f6c..5aba1c84dbec73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -21,22 +21,18 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.jupiter.api.Test; -import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentMap; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -53,109 +49,109 @@ public class TestDiagnosticJStackService { - private static final int NUMBER_OF_JSTACKS = 3; - private static final String DUMMY_JSTACK = - "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; - private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; - private static final ApplicationId APPLICATION_ID = - ApplicationId.fromString(APPLICATION_ID_STR); - private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; - private static final ContainerId CONTAINER_ID = - ContainerId.fromString(CONTAINER_ID_STR); - private static final ApplicationACLsManager mockAclManager = mock(ApplicationACLsManager.class); - - private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( - null, null, null - , mockAclManager, null, false, new Configuration() - ); - private static final DiagnosticJStackService diagnosticJStackService - = spy(new DiagnosticJStackService(nmContext)); - - - @Test - public void testWrongApplicationId() { - String applicationId = "app_29042"; - - assertThrows(RuntimeException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); + private static final int NUMBER_OF_JSTACKS = 3; + private static final String DUMMY_JSTACK = + "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; + private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; + private static final ApplicationId APPLICATION_ID = + ApplicationId.fromString(APPLICATION_ID_STR); + private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; + private static final ContainerId CONTAINER_ID = + ContainerId.fromString(CONTAINER_ID_STR); + private static final ApplicationACLsManager mockAclManager = + mock(ApplicationACLsManager.class); + + private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( + null, null, null + ,mockAclManager, null, false, new Configuration() + ); + private static final DiagnosticJStackService diagnosticJStackService = + spy(new DiagnosticJStackService(nmContext)); + + + @Test + public void testWrongApplicationId() { + String applicationId = "app_29042"; + + assertThrows(RuntimeException.class, + () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); + } + + @Test + public void testCollectNodeThreadDump_Success() { + // No need to mock ProcessID, as it will take the unit test JVM PID + try(MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + // Wrap mockConstruction here to automatically close it + ){ + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + when(mockAclManager.isAdmin(any())).thenReturn(true); + + String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + + assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated relative to Number of JStacks"); + + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } + + assertTrue(result.contains("--- JStack iteration 0")); + assertTrue(result.contains("--- JStack iteration 1")); + assertTrue(result.contains("--- JStack iteration 2")); + assertTrue(result.contains(DUMMY_JSTACK)); + + } catch (IOException e) { + throw new RuntimeException(e); + } } - @Test - public void testCollectNodeThreadDump_Success() { - // No need to mock ProcessID, as it will take the unit test JVM PID - try(MockedConstruction mockedConstruction = - mockConstruction(Shell.ShellCommandExecutor.class, - (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK) - ) // Wrap mockConstruction here to automatically close it - ){ - HttpServletRequest mockRequest = mock(HttpServletRequest.class); - when(mockAclManager.isAdmin(any())).thenReturn(true); + @Test + public void testCollectApplicationThreadDump_Success() { + List pids = List.of(23L, 12L, 531L); - String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + Application mockApp = mock(Application.class); + nmContext.getApplications().put(APPLICATION_ID, mockApp); - assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), - "ShellCommandExecutor should be instantiated relative to Number of JStacks"); + when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); - // Verify each individual mock was used exactly once - for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { - verify(mockExecutor, times(1)).execute(); - verify(mockExecutor, times(1)).getOutput(); - } + Map> containerPids = Map.of(CONTAINER_ID, pids); + doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); - assertTrue(result.contains("--- JStack iteration 0")); - assertTrue(result.contains("--- JStack iteration 1")); - assertTrue(result.contains("--- JStack iteration 2")); - assertTrue(result.contains(DUMMY_JSTACK)); + ProcessHandle mockProcessHandle = mock(ProcessHandle.class); + ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - public void testCollectApplicationThreadDump_Success() { - List pids = List.of(23L, 12L, 531L); + when(mockProcessHandle.info()).thenReturn(mockPhInfo); + when(mockPhInfo.user()).thenReturn(Optional.empty()); - Application mockApp = mock(Application.class); - nmContext.getApplications().put(APPLICATION_ID, mockApp); - - when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); - - Map> containerPids = Map.of(CONTAINER_ID, pids); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); - - ProcessHandle mockProcessHandle = mock(ProcessHandle.class); - ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); - - when(mockProcessHandle.info()).thenReturn(mockPhInfo); - when(mockPhInfo.user()).thenReturn(Optional.empty()); - - try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); - MockedConstruction mockedConstruction = - mockConstruction(Shell.ShellCommandExecutor.class, - (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); + MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) // Wrap mockedStatic & mockedConstruction here to automatically close them ){ - mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); + mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); - - assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), - "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); - // Verify each individual mock was used exactly once - for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { - verify(mockExecutor, times(1)).execute(); - verify(mockExecutor, times(1)).getOutput(); - } + assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); - assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); - assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); - assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); - assertTrue(result.contains(DUMMY_JSTACK)); + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } + assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); + assertTrue(result.contains(DUMMY_JSTACK)); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException(e); } nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test @@ -165,27 +161,27 @@ public void testCollectApplicationThreadDump_Success() { @Test public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { - int numJStacks = 3; - Application mockApp = mock(Application.class); - nmContext.getApplications().put(APPLICATION_ID, mockApp); + int numJStacks = 3; + Application mockApp = mock(Application.class); + nmContext.getApplications().put(APPLICATION_ID, mockApp); - when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); + when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); - Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); + Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); - doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); + doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); - assertNotNull(result); - assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), - "Output should contain the container ID"); - assertTrue(result.contains("Status: Skipped Process with PID"), - "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); + assertNotNull(result); + assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), + "Output should contain the container ID"); + assertTrue(result.contains("Status: Skipped Process with PID"), + "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); - nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test } From 48f5ad4a5547aa05b4dbb16ce2aed96e027fe72c Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Sun, 29 Mar 2026 17:15:22 +0200 Subject: [PATCH 29/35] YARN-11823: fix checkstyle and bug warning --- .../webapp/DiagnosticJStackService.java | 56 +++++----- .../nodemanager/webapp/NMWebServices.java | 10 +- .../webapp/TestDiagnosticJStackService.java | 101 +++++++++--------- 3 files changed, 83 insertions(+), 84 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index 930604b312e162..a72526e55fe8ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -55,8 +55,7 @@ public DiagnosticJStackService(Context context) { } public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req) - throws IOException - { + throws IOException { checkShellNotWindows(); long nodeManagerPid = ProcessHandle.current().pid(); @@ -78,8 +77,7 @@ private void checkAdminACL(HttpServletRequest req) throws IOException { } public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) - throws IOException - { + throws IOException { checkShellNotWindows(); ApplicationId applicationId = ApplicationId.fromString(appId); @@ -100,7 +98,7 @@ private void checkApplicationACL(HttpServletRequest req, Application app) throws UserGroupInformation callerUGI = getUserGroupInformation(req); boolean isAuthorized = context.getApplicationACLsManager().checkAccess( - callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId() + callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId() ); if(!isAuthorized){ @@ -124,17 +122,18 @@ protected Map> getApplicationContainerPids(Application a long parentPid = Long.parseLong(pidForContainerIdStr); List javaContainerPids = ProcessHandle.of(parentPid).stream() - .flatMap(ProcessHandle::descendants) - .filter(childProcess -> { - String cmdLine = childProcess.info().commandLine().orElse("").trim(); - // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk-cloudera/bin/java - // -Djava.net.preferIPv4Stack=true - if (cmdLine.isEmpty()){ - return false; - } - - String executable = cmdLine.split("\\s+")[0]; // The first token is always the executable binary - return executable.equals("java") || executable.endsWith("/java"); + .flatMap(ProcessHandle::descendants) + .filter(childProcess -> { + String cmdLine = childProcess.info().commandLine().orElse("").trim(); + // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk-cloudera/bin/java + // -Djava.net.preferIPv4Stack=true + if (cmdLine.isEmpty()){ + return false; + } + + String executable = cmdLine.split("\\s+")[0]; + // The first token is always the executable binary + return executable.equals("java") || executable.endsWith("/java"); }) .map(ProcessHandle::pid) .toList(); @@ -151,17 +150,18 @@ protected Map> getApplicationContainerPids(Application a private String runJStack(Map> containerPids, int numJStacks){ StringBuilder result = new StringBuilder(); - for(ContainerId containerPid : containerPids.keySet()){ - List javaContainerPids = containerPids.get(containerPid); + for(Map.Entry> entry : containerPids.entrySet()){ + ContainerId containerId = entry.getKey(); + List javaContainerPids = entry.getValue(); if (javaContainerPids.isEmpty()){ - result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " + - "because no Java Process ID exist ===", containerPid.toString())); + result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " + + "because no Java Process ID exist ===", containerId.toString())); } else { for (Long pid : javaContainerPids) { result.append(String.format( - "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", - containerPid.toString(), pid, runJStack(pid, numJStacks))); + "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", + containerId.toString(), pid, runJStack(pid, numJStacks))); } } @@ -183,10 +183,10 @@ private String runJStack(long pid, int numJStacks) { String runningUser = processHandle.info().user().orElse(NM_USER); String containerExecutorPath = - PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); + PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); String[] jstackCommand = { - containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH + containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH }; LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); @@ -195,16 +195,16 @@ private String runJStack(long pid, int numJStacks) { for (int i = 0; i < numJStacks; i++) { Shell.ShellCommandExecutor cmd = - new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); + new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); try { cmd.execute(); result.append(String.format( - "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); } catch (IOException e) { result.append(String.format( - "Failed to run jstack on PID: " + pid + " at iteration: " + i + - " (Process likely exited before/during running jstack): " + e.getMessage())); + "Failed to run jstack on PID: " + pid + " at iteration: " + i + + " (Process likely exited before/during running jstack): " + e.getMessage())); break; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index c54e3c9090203f..d544893d7faeff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfoes; -import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -642,8 +641,7 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context @Path("/jstack/{numberOfJStack}") @Produces({ MediaType.TEXT_PLAIN}) public Response getNodeThreadDump(@javax.ws.rs.core.Context HttpServletRequest req, - @PathParam("numberOfJStack") int numberOfJStack) - { + @PathParam("numberOfJStack") int numberOfJStack) { if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) .build(); @@ -672,8 +670,7 @@ public Response getNodeThreadDump(@javax.ws.rs.core.Context HttpServletRequest r @Produces({MediaType.TEXT_PLAIN}) public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletRequest req, @PathParam("appid") String appId, - @PathParam("numberOfJStack") int numberOfJStack) - { + @PathParam("numberOfJStack") int numberOfJStack) { if (!isJStackEndpointsEnable) { return Response.status(Status.METHOD_NOT_ALLOWED) .build(); @@ -681,7 +678,8 @@ public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletReques try { return Response.status(Status.OK) - .entity(diagnosticJStackService.collectApplicationThreadDump(appId, numberOfJStack, req)) + .entity(diagnosticJStackService + .collectApplicationThreadDump(appId, numberOfJStack, req)) .build(); } catch (IllegalArgumentException e){ return Response.status(Status.BAD_REQUEST) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 5aba1c84dbec73..9da14f8b3be976 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -51,37 +51,37 @@ public class TestDiagnosticJStackService { private static final int NUMBER_OF_JSTACKS = 3; private static final String DUMMY_JSTACK = - "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; + "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; private static final ApplicationId APPLICATION_ID = - ApplicationId.fromString(APPLICATION_ID_STR); + ApplicationId.fromString(APPLICATION_ID_STR); private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; private static final ContainerId CONTAINER_ID = - ContainerId.fromString(CONTAINER_ID_STR); + ContainerId.fromString(CONTAINER_ID_STR); private static final ApplicationACLsManager mockAclManager = - mock(ApplicationACLsManager.class); + mock(ApplicationACLsManager.class); private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( - null, null, null - ,mockAclManager, null, false, new Configuration() + null, null, null + ,mockAclManager, null, false, new Configuration() ); private static final DiagnosticJStackService diagnosticJStackService = - spy(new DiagnosticJStackService(nmContext)); + spy(new DiagnosticJStackService(nmContext)); @Test public void testWrongApplicationId() { - String applicationId = "app_29042"; + String applicationId = "app_29042"; - assertThrows(RuntimeException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); + assertThrows(RuntimeException.class, + () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); } @Test - public void testCollectNodeThreadDump_Success() { + public void testCollectNodeThreadDumpSuccess() { // No need to mock ProcessID, as it will take the unit test JVM PID try(MockedConstruction mockedConstruction = - mockConstruction(Shell.ShellCommandExecutor.class, + mockConstruction(Shell.ShellCommandExecutor.class, (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) // Wrap mockConstruction here to automatically close it ){ @@ -91,7 +91,7 @@ public void testCollectNodeThreadDump_Success() { String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), - "ShellCommandExecutor should be instantiated relative to Number of JStacks"); + "ShellCommandExecutor should be instantiated relative to Number of JStacks"); // Verify each individual mock was used exactly once for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { @@ -104,13 +104,13 @@ public void testCollectNodeThreadDump_Success() { assertTrue(result.contains("--- JStack iteration 2")); assertTrue(result.contains(DUMMY_JSTACK)); - } catch (IOException e) { - throw new RuntimeException(e); - } + } catch (IOException e) { + throw new RuntimeException(e); } + } @Test - public void testCollectApplicationThreadDump_Success() { + public void testCollectApplicationThreadDumpSuccess() { List pids = List.of(23L, 12L, 531L); Application mockApp = mock(Application.class); @@ -128,62 +128,63 @@ public void testCollectApplicationThreadDump_Success() { when(mockPhInfo.user()).thenReturn(Optional.empty()); try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); - MockedConstruction mockedConstruction = - mockConstruction(Shell.ShellCommandExecutor.class, - (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) // Wrap mockedStatic & mockedConstruction here to automatically close them ){ - mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); + mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); - assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); - // Verify each individual mock was used exactly once - for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { verify(mockExecutor, times(1)).execute(); verify(mockExecutor, times(1)).getOutput(); - } - assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); - assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); - assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); - assertTrue(result.contains(DUMMY_JSTACK)); - } catch (IOException e) { + } + + assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); + assertTrue(result.contains(DUMMY_JSTACK)); + } catch (IOException e) { throw new RuntimeException(e); - } + } - nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test - } + } - @Test - public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { - int numJStacks = 3; - Application mockApp = mock(Application.class); - nmContext.getApplications().put(APPLICATION_ID, mockApp); + @Test + public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { + int numJStacks = 3; + Application mockApp = mock(Application.class); + nmContext.getApplications().put(APPLICATION_ID, mockApp); - when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); + when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); - Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); + Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); - doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); + doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); - assertNotNull(result); - assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), + assertNotNull(result); + assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), "Output should contain the container ID"); - assertTrue(result.contains("Status: Skipped Process with PID"), + assertTrue(result.contains("Status: Skipped Process with PID"), "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); - nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test - } + } From 391abdd15887ff581eab37f8bb371f0a8bc84aa6 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Fri, 3 Apr 2026 18:17:17 +0200 Subject: [PATCH 30/35] YARN-11823: Fix checkstyle --- .../webapp/DiagnosticJStackService.java | 13 ++--- .../webapp/TestDiagnosticJStackService.java | 49 ++++++++++--------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index a72526e55fe8ab..fe5fca9dae750a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -76,7 +76,8 @@ private void checkAdminACL(HttpServletRequest req) throws IOException { } } - public String collectApplicationThreadDump(String appId, int numberOfJStack, HttpServletRequest req) + public String collectApplicationThreadDump( + String appId, int numberOfJStack, HttpServletRequest req) throws IOException { checkShellNotWindows(); @@ -125,7 +126,7 @@ protected Map> getApplicationContainerPids(Application a .flatMap(ProcessHandle::descendants) .filter(childProcess -> { String cmdLine = childProcess.info().commandLine().orElse("").trim(); - // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk-cloudera/bin/java + // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk/bin/java // -Djava.net.preferIPv4Stack=true if (cmdLine.isEmpty()){ return false; @@ -134,9 +135,9 @@ protected Map> getApplicationContainerPids(Application a String executable = cmdLine.split("\\s+")[0]; // The first token is always the executable binary return executable.equals("java") || executable.endsWith("/java"); - }) - .map(ProcessHandle::pid) - .toList(); + }) + .map(ProcessHandle::pid) + .toList(); containerPids.put(containerId, javaContainerPids); @@ -156,7 +157,7 @@ private String runJStack(Map> containerPids, int numJSta if (javaContainerPids.isEmpty()){ result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " + - "because no Java Process ID exist ===", containerId.toString())); + "because no Java Process ID exist ===", containerId.toString())); } else { for (Long pid : javaContainerPids) { result.append(String.format( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 9da14f8b3be976..d0e76075bb268e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -58,14 +58,14 @@ public class TestDiagnosticJStackService { private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; private static final ContainerId CONTAINER_ID = ContainerId.fromString(CONTAINER_ID_STR); - private static final ApplicationACLsManager mockAclManager = + private static final ApplicationACLsManager MOCK_ACLS_MANAGER = mock(ApplicationACLsManager.class); private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( - null, null, null - ,mockAclManager, null, false, new Configuration() + null ,null ,null + ,MOCK_ACLS_MANAGER ,null ,false ,new Configuration() ); - private static final DiagnosticJStackService diagnosticJStackService = + private static final DiagnosticJStackService DIAGNOSTIC_JSTACK_SERVICE = spy(new DiagnosticJStackService(nmContext)); @@ -74,7 +74,7 @@ public void testWrongApplicationId() { String applicationId = "app_29042"; assertThrows(RuntimeException.class, - () -> diagnosticJStackService.collectApplicationThreadDump(applicationId, 3, null)); + () -> DIAGNOSTIC_JSTACK_SERVICE.collectApplicationThreadDump(applicationId, 3, null)); } @Test @@ -82,13 +82,13 @@ public void testCollectNodeThreadDumpSuccess() { // No need to mock ProcessID, as it will take the unit test JVM PID try(MockedConstruction mockedConstruction = mockConstruction(Shell.ShellCommandExecutor.class, - (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) // Wrap mockConstruction here to automatically close it ){ HttpServletRequest mockRequest = mock(HttpServletRequest.class); - when(mockAclManager.isAdmin(any())).thenReturn(true); + when(MOCK_ACLS_MANAGER.isAdmin(any())).thenReturn(true); - String result = diagnosticJStackService.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + String result = DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated relative to Number of JStacks"); @@ -116,10 +116,10 @@ public void testCollectApplicationThreadDumpSuccess() { Application mockApp = mock(Application.class); nmContext.getApplications().put(APPLICATION_ID, mockApp); - when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); + when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); Map> containerPids = Map.of(CONTAINER_ID, pids); - doReturn(containerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); + doReturn(containerPids).when(DIAGNOSTIC_JSTACK_SERVICE).getApplicationContainerPids(mockApp); ProcessHandle mockProcessHandle = mock(ProcessHandle.class); ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); @@ -130,21 +130,23 @@ public void testCollectApplicationThreadDumpSuccess() { try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); MockedConstruction mockedConstruction = mockConstruction(Shell.ShellCommandExecutor.class, - (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) // Wrap mockedStatic & mockedConstruction here to automatically close them ){ - mockedStaticProcess.when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); + mockedStaticProcess + .when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); + String result = DIAGNOSTIC_JSTACK_SERVICE + .collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); // Verify each individual mock was used exactly once for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { - verify(mockExecutor, times(1)).execute(); - verify(mockExecutor, times(1)).getOutput(); + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); } assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); @@ -152,10 +154,11 @@ public void testCollectApplicationThreadDumpSuccess() { assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); assertTrue(result.contains(DUMMY_JSTACK)); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException(e); } - nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + nmContext.getApplications().remove(APPLICATION_ID); + // Clean up to avoid side effects on another test } @@ -166,23 +169,25 @@ public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOExc Application mockApp = mock(Application.class); nmContext.getApplications().put(APPLICATION_ID, mockApp); - when(mockAclManager.checkAccess(any(), any(), any(), any())).thenReturn(true); + when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); - doReturn(fakeContainerPids).when(diagnosticJStackService).getApplicationContainerPids(mockApp); + doReturn(fakeContainerPids).when(DIAGNOSTIC_JSTACK_SERVICE).getApplicationContainerPids(mockApp); HttpServletRequest mockRequest = mock(HttpServletRequest.class); - String result = diagnosticJStackService.collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + String result = DIAGNOSTIC_JSTACK_SERVICE + .collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); assertNotNull(result); assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), "Output should contain the container ID"); assertTrue(result.contains("Status: Skipped Process with PID"), - "Since we did not mock ProcessHandle.of to return non empty, it will consider this PID is dead"); + "Since we don't mock ProcessHandle.of to return non empty it considers this PID is dead"); - nmContext.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test + nmContext.getApplications().remove(APPLICATION_ID); + // Clean up to avoid side effects on another test } From 76d99735c5587b6680fc1e6c634b228b4cab331f Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 23 Apr 2026 17:01:17 +0200 Subject: [PATCH 31/35] YARN-11823: Add documentation for the two new endpoints and remove some comments --- .../webapp/DiagnosticJStackService.java | 3 - .../src/site/markdown/NodeManagerRest.md | 112 ++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java index fe5fca9dae750a..0cbaad9371f998 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -126,12 +126,9 @@ protected Map> getApplicationContainerPids(Application a .flatMap(ProcessHandle::descendants) .filter(childProcess -> { String cmdLine = childProcess.info().commandLine().orElse("").trim(); - // Command Line: /usr/lib/jvm/jdk1.17.0.11.0-openjdk/bin/java - // -Djava.net.preferIPv4Stack=true if (cmdLine.isEmpty()){ return false; } - String executable = cmdLine.split("\\s+")[0]; // The first token is always the executable binary return executable.equals("java") || executable.endsWith("/java"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md index b088c48dac70d5..44f1d1f0ebfa9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md @@ -787,3 +787,115 @@ Response Body: "assignedGpuDevices": [] } ``` + +JStack NodeManager API +---------------- +With JStack NodeManager API, you can get the JStack of the NodeManager processID. + +### URI + +Use the following URI to obtain resources on the NodeManager + + * http://nm-http-address:port/ws/v1/node/jstack/{numberOfJStack} + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + + None + +### GET Response Examples + +**PLAIN TEXT response** + +HTTP Request: + + GET http://nm-http-address:port/ws/v1/node/jstack/{numberOfJStack} + +Response Header: + + Cache-Control: no-cache + Pragma: no-cache + X-Content-Type-Options: nosniff + X-XSS-Protection: 1; mode=block + X-Frame-Options: SAMEORIGIN + Content-Type: text/plain + Vary: Accept-Encoding + Content-Encoding: gzip + Transfer-Encoding: chunked + + +```text +--- JStack iteration 0 for PID: 322091 --- +2026-04-23 14:39:46 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +............... +............... +JNI global refs: 66, weak refs: 0 +--- JStack iteration 1 for PID: 322091 --- +2026-04-23 14:39:46 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +``` + +JStack Application API +---------------- +With JStack Application API, you can get the JStack of all java process Containers that belongs to the ApplicationId + +### URI + +Use the following URI to obtain resources on the NodeManager + + * http://nm-http-address:port/ws/v1/node/apps/{appid}/jstack/{numberOfJStack} + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + + None + +### GET Response Examples + +**PLAIN TEXT response** + +HTTP Request: + + GET http://nm-http-address:port/ws/v1/node/apps/{appid}/jstack/{numberOfJStack} + +Response Header: + + Cache-Control: no-cache + Pragma: no-cache + X-Content-Type-Options: nosniff + X-XSS-Protection: 1; mode=block + X-Frame-Options: SAMEORIGIN + Content-Type: text/plain + Vary: Accept-Encoding + Content-Encoding: gzip + Transfer-Encoding: chunked + + +```text +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000001, PID: 358474 === +--- JStack iteration 0 for PID: 358474 --- +2026-04-23 14:54:47 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +............... +............... +Failed to run jstack on PID: 362652 at iteration: 3 (Process likely exited before/during running jstack): Exception in thread "main" java.io.IOException: Premature EOF + at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.readInt(HotSpotVirtualMachine.java:341) + at jdk.attach/sun.tools.attach.VirtualMachineImpl.execute(VirtualMachineImpl.java:197) + at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.executeCommand(HotSpotVirtualMachine.java:310) + at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.remoteDataDump(HotSpotVirtualMachine.java:267) + at jdk.jcmd/sun.tools.jstack.JStack.runThreadDump(JStack.java:130) + at jdk.jcmd/sun.tools.jstack.JStack.main(JStack.java:109) + +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000038, PID: 362667 === +Failed to run jstack on PID: 362667 at iteration: 0 (Process likely exited before/during running jstack): 362667: No such process + +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000039, PID: 362658 === +Status: Skipped Process with PID Process with PID 362658 is no longer exists +``` \ No newline at end of file From 563d26eafce795476af24c90ee4314d564cc21a3 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 23 Apr 2026 23:36:10 +0200 Subject: [PATCH 32/35] YARN-11823: Fixed check style problem --- .../webapp/TestDiagnosticJStackService.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index d0e76075bb268e..24807cf66b6730 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -61,12 +61,12 @@ public class TestDiagnosticJStackService { private static final ApplicationACLsManager MOCK_ACLS_MANAGER = mock(ApplicationACLsManager.class); - private static final NodeManager.NMContext nmContext = new NodeManager.NMContext( - null ,null ,null - ,MOCK_ACLS_MANAGER ,null ,false ,new Configuration() + private static final NodeManager.NMContext NM_CONTEXT = new NodeManager.NMContext( + null, null, null + , MOCK_ACLS_MANAGER , null , false, new Configuration() ); private static final DiagnosticJStackService DIAGNOSTIC_JSTACK_SERVICE = - spy(new DiagnosticJStackService(nmContext)); + spy(new DiagnosticJStackService(NM_CONTEXT)); @Test @@ -88,7 +88,8 @@ public void testCollectNodeThreadDumpSuccess() { HttpServletRequest mockRequest = mock(HttpServletRequest.class); when(MOCK_ACLS_MANAGER.isAdmin(any())).thenReturn(true); - String result = DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + String result = + DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated relative to Number of JStacks"); @@ -114,7 +115,7 @@ public void testCollectApplicationThreadDumpSuccess() { List pids = List.of(23L, 12L, 531L); Application mockApp = mock(Application.class); - nmContext.getApplications().put(APPLICATION_ID, mockApp); + NM_CONTEXT.getApplications().put(APPLICATION_ID, mockApp); when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); @@ -138,7 +139,7 @@ public void testCollectApplicationThreadDumpSuccess() { HttpServletRequest mockRequest = mock(HttpServletRequest.class); String result = DIAGNOSTIC_JSTACK_SERVICE - .collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); + .collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); @@ -157,7 +158,7 @@ public void testCollectApplicationThreadDumpSuccess() { throw new RuntimeException(e); } - nmContext.getApplications().remove(APPLICATION_ID); + NM_CONTEXT.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test } @@ -167,18 +168,19 @@ public void testCollectApplicationThreadDumpSuccess() { public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { int numJStacks = 3; Application mockApp = mock(Application.class); - nmContext.getApplications().put(APPLICATION_ID, mockApp); + NM_CONTEXT.getApplications().put(APPLICATION_ID, mockApp); when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); - doReturn(fakeContainerPids).when(DIAGNOSTIC_JSTACK_SERVICE).getApplicationContainerPids(mockApp); + doReturn(fakeContainerPids).when(DIAGNOSTIC_JSTACK_SERVICE) + .getApplicationContainerPids(mockApp); HttpServletRequest mockRequest = mock(HttpServletRequest.class); String result = DIAGNOSTIC_JSTACK_SERVICE - .collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + .collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); assertNotNull(result); assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), @@ -186,7 +188,7 @@ public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOExc assertTrue(result.contains("Status: Skipped Process with PID"), "Since we don't mock ProcessHandle.of to return non empty it considers this PID is dead"); - nmContext.getApplications().remove(APPLICATION_ID); + NM_CONTEXT.getApplications().remove(APPLICATION_ID); // Clean up to avoid side effects on another test } From a400f5dd9962edddb27eb577cd96e7e521d6a37b Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Mon, 27 Apr 2026 19:08:26 +0200 Subject: [PATCH 33/35] YARN-11823: Fixed checkstyle --- .../nodemanager/webapp/TestDiagnosticJStackService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 24807cf66b6730..43208f38292c66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -62,8 +62,8 @@ public class TestDiagnosticJStackService { mock(ApplicationACLsManager.class); private static final NodeManager.NMContext NM_CONTEXT = new NodeManager.NMContext( - null, null, null - , MOCK_ACLS_MANAGER , null , false, new Configuration() + null , null , null + , MOCK_ACLS_MANAGER , null , false , new Configuration() ); private static final DiagnosticJStackService DIAGNOSTIC_JSTACK_SERVICE = spy(new DiagnosticJStackService(NM_CONTEXT)); @@ -89,7 +89,7 @@ public void testCollectNodeThreadDumpSuccess() { when(MOCK_ACLS_MANAGER.isAdmin(any())).thenReturn(true); String result = - DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), "ShellCommandExecutor should be instantiated relative to Number of JStacks"); From 3afde292baaa1de231175654bb5c6fa8300f2377 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Tue, 28 Apr 2026 09:44:14 +0200 Subject: [PATCH 34/35] YARN-11823: Fix checkstyle indentation --- .../webapp/TestDiagnosticJStackService.java | 4 ++-- .../src/site/markdown/NodeManagerRest.md | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java index 43208f38292c66..54af10ba927ab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -62,8 +62,8 @@ public class TestDiagnosticJStackService { mock(ApplicationACLsManager.class); private static final NodeManager.NMContext NM_CONTEXT = new NodeManager.NMContext( - null , null , null - , MOCK_ACLS_MANAGER , null , false , new Configuration() + null, null, null, + MOCK_ACLS_MANAGER, null, false, new Configuration() ); private static final DiagnosticJStackService DIAGNOSTIC_JSTACK_SERVICE = spy(new DiagnosticJStackService(NM_CONTEXT)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md index 44f1d1f0ebfa9d..75c5feb3f0904f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md @@ -886,12 +886,12 @@ Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mod ............... ............... Failed to run jstack on PID: 362652 at iteration: 3 (Process likely exited before/during running jstack): Exception in thread "main" java.io.IOException: Premature EOF - at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.readInt(HotSpotVirtualMachine.java:341) - at jdk.attach/sun.tools.attach.VirtualMachineImpl.execute(VirtualMachineImpl.java:197) - at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.executeCommand(HotSpotVirtualMachine.java:310) - at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.remoteDataDump(HotSpotVirtualMachine.java:267) - at jdk.jcmd/sun.tools.jstack.JStack.runThreadDump(JStack.java:130) - at jdk.jcmd/sun.tools.jstack.JStack.main(JStack.java:109) +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.readInt(HotSpotVirtualMachine.java:341) +at jdk.attach/sun.tools.attach.VirtualMachineImpl.execute(VirtualMachineImpl.java:197) +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.executeCommand(HotSpotVirtualMachine.java:310) +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.remoteDataDump(HotSpotVirtualMachine.java:267) +at jdk.jcmd/sun.tools.jstack.JStack.runThreadDump(JStack.java:130) +at jdk.jcmd/sun.tools.jstack.JStack.main(JStack.java:109) === Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000038, PID: 362667 === Failed to run jstack on PID: 362667 at iteration: 0 (Process likely exited before/during running jstack): 362667: No such process From 4af108911af511338d6b91471a36452242f8ebe3 Mon Sep 17 00:00:00 2001 From: Hean-Chhinling Date: Thu, 3 Sep 2026 16:00:58 +0200 Subject: [PATCH 35/35] YARN-11823: Add a unit-test and display the command --- .../impl/container-executor.c | 11 ++--- .../impl/container-executor.h | 4 +- .../native/container-executor/impl/main.c | 3 ++ .../test/test-container-executor.c | 46 +++++++++++++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index ee7f72ef2e58d7..e8650c8235bc05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -3311,18 +3311,15 @@ int remove_docker_container(char**argv, int argc) { return exit_code; } -int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path){ +int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path) { int exit_code = set_user(user); if (exit_code != 0) { - fprintf(ERRORFILE, "Failed to set user to %s\n", user); - return exit_code; + fprintf(ERRORFILE, "Failed to set user to %s\n", user); + return exit_code; } - // Use exec "$0" "$1" to treat as positional arguments, - // preventing command injection vulnerability CVE-2023-25555 - execlp("/bin/bash", "bash", "-c", "exec \"$0\" \"$1\"", jstack_path, pid, NULL); + execl(jstack_path, "jstack", pid, (char *) NULL); fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno)); return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 4327d939ac33c2..58c8ad561a8bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -371,6 +371,6 @@ int is_terminal_support_enabled(); int is_runc_support_enabled(); /** -* Run jstack as a specific user -*/ + * Run jstack as a specific user. + */ int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 6318b99e18bdba..0e1b4d0e1040f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -65,6 +65,9 @@ static void display_usage(FILE *stream) { fprintf(stream, "%s container-executor --reap-runc-layer-mounts \n", de); + fprintf(stream, + " container-executor --run-jstack \n"); + fprintf(stream, " container-executor \n" " where command and command-args: \n" \ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index f209ea53a00963..bb51a05a1f08ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -1383,6 +1383,51 @@ void test_is_empty() { } } +static int create_mock_jstack(const char *path) { + FILE *file = fopen(path, "w"); + if (file == NULL) { + return -1; + } + fprintf(file, "#!/bin/sh\necho mock-jstack\nexit 0\n"); + fclose(file); + return chmod(path, 0755); +} + +void test_run_jstack_as_user() { + char jstack_path[PATH_MAX]; + char pid_buf[32]; + pid_t child; + int status = 0; + + printf("\nTesting run_jstack_as_user\n"); + snprintf(jstack_path, sizeof(jstack_path), "%s/jstack", TEST_ROOT); + if (create_mock_jstack(jstack_path) != 0) { + printf("FAIL: could not create mock jstack at %s\n", jstack_path); + exit(1); + } + snprintf(pid_buf, sizeof(pid_buf), "%d", getpid()); + + child = fork(); + if (child == -1) { + printf("FAIL: failed to fork for run_jstack_as_user test - %s\n", + strerror(errno)); + exit(1); + } + if (child == 0) { + _exit(run_jstack_as_user(username, pid_buf, jstack_path)); + } + if (waitpid(child, &status, 0) <= 0) { + printf("FAIL: failed waiting for run_jstack_as_user child - %s\n", + strerror(errno)); + exit(1); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + printf("FAIL: run_jstack_as_user child exited with status %d\n", + WIFEXITED(status) ? WEXITSTATUS(status) : -1); + exit(1); + } +} + #define TCE_FAKE_CGROOT TEST_ROOT "/cgroup_root" #define TCE_NUM_CG_CONTROLLERS 6 extern int clean_docker_cgroups_internal(const char *mount_table, @@ -1803,6 +1848,7 @@ int main(int argc, char **argv) { test_trim_function(); test_concatenate(); + test_run_jstack_as_user(); printf("\nFinished tests\n"); printf("\nAttempting to clean up from the run\n");