diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6871e4b2a219c0..2b0d945d5bde79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2325,6 +2325,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; + /** Flag to turn on/off jstack endpoints for NodeManager. By default is False **/ + public static final String NM_JSTACK_ENDPOINTS_ENABLED = + NM_PREFIX + "jstack-endpoints.enabled"; + public static final boolean DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED = false; + /** * Comma separated list of runtimes that are allowed when using * LinuxContainerExecutor. The standard values are: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 830cade703dee7..a9c766ca60c89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5856,6 +5856,17 @@ 50000 + + + + This configuration will be used to turn on/turn off nodemanager jstack endpoints + /ws/v1/node/jstack/{numberOfJStack} and /ws/v1/node/apps/{appid}/jstack/{numberOfJStack}. + The purpose is to minimise security risk. By default is set to false. + + yarn.nodemanager.jstack-endpoints.enabled + false + + Set the connect timeout interval, in milliseconds. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java new file mode 100644 index 00000000000000..0cbaad9371f998 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/DiagnosticJStackService.java @@ -0,0 +1,229 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.util.Map; +import java.util.List; +import java.util.HashMap; +import java.util.Optional; +import java.util.Arrays; + +public class DiagnosticJStackService { + + private static final Logger LOG = LoggerFactory.getLogger(DiagnosticJStackService.class); + + private static final String NM_USER = System.getProperty("user.name"); + private static final String JSTACK_PATH = System.getProperty("java.home") + "/bin/jstack"; + private final Context context; + private final Configuration conf; + + public DiagnosticJStackService(Context context) { + this.context = context; + this.conf = context.getConf(); + } + + public String collectNodeThreadDump(int numberOfJStack, HttpServletRequest req) + throws IOException { + checkShellNotWindows(); + + long nodeManagerPid = ProcessHandle.current().pid(); + + checkAdminACL(req); + + return runJStack(nodeManagerPid, numberOfJStack); + } + + private void checkAdminACL(HttpServletRequest req) throws IOException { + UserGroupInformation callerUGI = getUserGroupInformation(req); + + boolean isAdmin = context.getApplicationACLsManager().isAdmin(callerUGI); + + if (!isAdmin) { + throw new YarnRuntimeException("User " + callerUGI.getShortUserName() + + " is not authorized to run jstack on NodeManager "); + } + } + + public String collectApplicationThreadDump( + String appId, int numberOfJStack, HttpServletRequest req) + throws IOException { + checkShellNotWindows(); + + ApplicationId applicationId = ApplicationId.fromString(appId); + + Application app = context.getApplications().get(applicationId); + if (app == null){ + throw new YarnRuntimeException("Application " + applicationId + " does not exist"); + } + + checkApplicationACL(req, app); + + Map> containerPids = getApplicationContainerPids(app); + + return runJStack(containerPids, numberOfJStack); + } + + private void checkApplicationACL(HttpServletRequest req, Application app) throws IOException { + UserGroupInformation callerUGI = getUserGroupInformation(req); + + boolean isAuthorized = context.getApplicationACLsManager().checkAccess( + callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getAppId() + ); + + if(!isAuthorized){ + throw new YarnRuntimeException("User " + callerUGI.getShortUserName() + + " is not authorized to view application " + app.getAppId()); + } + + } + + private void checkShellNotWindows() { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); + } + } + + protected Map> getApplicationContainerPids(Application app){ + Map> containerPids = new HashMap<>(); + + for (ContainerId containerId : app.getContainers().keySet()){ + String pidForContainerIdStr = context.getContainerExecutor().getProcessId(containerId); + long parentPid = Long.parseLong(pidForContainerIdStr); + + List javaContainerPids = ProcessHandle.of(parentPid).stream() + .flatMap(ProcessHandle::descendants) + .filter(childProcess -> { + String cmdLine = childProcess.info().commandLine().orElse("").trim(); + if (cmdLine.isEmpty()){ + return false; + } + String executable = cmdLine.split("\\s+")[0]; + // The first token is always the executable binary + return executable.equals("java") || executable.endsWith("/java"); + }) + .map(ProcessHandle::pid) + .toList(); + + containerPids.put(containerId, javaContainerPids); + + } + + LOG.info("Application PIDs by ContainerId: {}", containerPids); + + return containerPids; + } + + private String runJStack(Map> containerPids, int numJStacks){ + StringBuilder result = new StringBuilder(); + + for(Map.Entry> entry : containerPids.entrySet()){ + ContainerId containerId = entry.getKey(); + List javaContainerPids = entry.getValue(); + + if (javaContainerPids.isEmpty()){ + result.append(String.format("=== Thread Dumps for ContainerId: %s%n is skipped " + + "because no Java Process ID exist ===", containerId.toString())); + } else { + for (Long pid : javaContainerPids) { + result.append(String.format( + "=== Thread Dumps for ContainerId: %s, PID: %d ===%n%s%n", + containerId.toString(), pid, runJStack(pid, numJStacks))); + } + } + + } + + return result.toString(); + } + + private String runJStack(long pid, int numJStacks) { + Optional processHandleOpt = ProcessHandle.of(pid); + + if (processHandleOpt.isEmpty()){ + String msg = String.format("Process with PID " + pid + " is no longer exists"); + LOG.warn(msg); + return "Status: Skipped Process with PID " + msg; + } + + ProcessHandle processHandle = processHandleOpt.get(); + + String runningUser = processHandle.info().user().orElse(NM_USER); + String containerExecutorPath = + PrivilegedOperationExecutor.getContainerExecutorExecutablePath(conf); + + String[] jstackCommand = { + containerExecutorPath, "--run-jstack", runningUser, String.valueOf(pid), JSTACK_PATH + }; + + LOG.info("Running JStack command: {}", Arrays.toString(jstackCommand)); + + StringBuilder result = new StringBuilder(); + + for (int i = 0; i < numJStacks; i++) { + Shell.ShellCommandExecutor cmd = + new Shell.ShellCommandExecutor(jstackCommand, null, null, 60_000); + + try { + cmd.execute(); + result.append(String.format( + "--- JStack iteration %d for PID: %d ---%n%s%n", i, pid, cmd.getOutput())); + } catch (IOException e) { + result.append(String.format( + "Failed to run jstack on PID: " + pid + " at iteration: " + i + + " (Process likely exited before/during running jstack): " + e.getMessage())); + break; + } + } + + return result.toString(); + } + + private UserGroupInformation getUserGroupInformation(HttpServletRequest req) throws IOException { + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI; + + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } else { + callerUGI = UserGroupInformation.getCurrentUser(); // Fallback to current OS user + } + + LOG.info("Checking ACL for Caller UGI: {}", callerUGI.toString()); + + return callerUGI; + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 25a7c91a020882..d544893d7faeff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; @@ -107,7 +108,9 @@ public class NMWebServices { private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private String redirectWSUrl; private LogAggregationFileControllerFactory factory; + private DiagnosticJStackService diagnosticJStackService; private boolean filterAppsByUser = false; + private boolean isJStackEndpointsEnable = false; @javax.ws.rs.core.Context private HttpServletRequest request; @@ -132,6 +135,11 @@ public NMWebServices(final @javax.inject.Named("nm") Context nm, this.filterAppsByUser = this.nmContext.getConf().getBoolean( YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.isJStackEndpointsEnable = this.nmContext.getConf().getBoolean( + YarnConfiguration.NM_JSTACK_ENDPOINTS_ENABLED, + YarnConfiguration.DEFAULT_NM_JSTACK_ENDPOINTS_ENABLED + ); + this.diagnosticJStackService = new DiagnosticJStackService(this.nmContext); } public NMWebServices(final Context nm, final ResourceView view, @@ -628,6 +636,68 @@ public Response syncYarnSysFS(@javax.ws.rs.core.Context return Response.ok().build(); } + + @GET + @Path("/jstack/{numberOfJStack}") + @Produces({ MediaType.TEXT_PLAIN}) + public Response getNodeThreadDump(@javax.ws.rs.core.Context HttpServletRequest req, + @PathParam("numberOfJStack") int numberOfJStack) { + if (!isJStackEndpointsEnable) { + return Response.status(Status.METHOD_NOT_ALLOWED) + .build(); + } + + try { + return Response.status(Status.OK) + .entity(diagnosticJStackService.collectNodeThreadDump(numberOfJStack, req)) + .build(); + } catch (YarnRuntimeException e) { + return Response.status(Status.FORBIDDEN) + .entity(e.getMessage()) + .build(); + } catch (IOException e){ + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs.") + .build(); + } + + } + + + @GET + @Path("/apps/{appid}/jstack/{numberOfJStack}") + @Produces({MediaType.TEXT_PLAIN}) + public Response getApplicationJStack(@javax.ws.rs.core.Context HttpServletRequest req, + @PathParam("appid") String appId, + @PathParam("numberOfJStack") int numberOfJStack) { + if (!isJStackEndpointsEnable) { + return Response.status(Status.METHOD_NOT_ALLOWED) + .build(); + } + + try { + return Response.status(Status.OK) + .entity(diagnosticJStackService + .collectApplicationThreadDump(appId, numberOfJStack, req)) + .build(); + } catch (IllegalArgumentException e){ + return Response.status(Status.BAD_REQUEST) + .entity("The applicationId is invalid: " + appId + ". " + e.getMessage()) + .build(); + } catch (YarnRuntimeException e) { + return Response.status(Status.FORBIDDEN) + .entity(e.getMessage()) + .build(); + } catch (IOException e){ + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity("Shell command has failed: " + e.getMessage() + ". " + + "For more information please check the NodeManager logs.") + .build(); + } + + } + private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 57fd58494b14f6..e8650c8235bc05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -3310,3 +3310,16 @@ int remove_docker_container(char**argv, int argc) { } return exit_code; } + +int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path) { + int exit_code = set_user(user); + if (exit_code != 0) { + fprintf(ERRORFILE, "Failed to set user to %s\n", user); + return exit_code; + } + + execl(jstack_path, "jstack", pid, (char *) NULL); + + fprintf(LOGFILE, "Failed to execute jstack: %s\n", strerror(errno)); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 8219a6755006eb..58c8ad561a8bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -54,7 +54,8 @@ enum operations { RUN_AS_USER_SYNC_YARN_SYSFS = 15, EXEC_CONTAINER = 16, RUN_RUNC_CONTAINER = 17, - REAP_RUNC_LAYER_MOUNTS = 18 + REAP_RUNC_LAYER_MOUNTS = 18, + RUN_JSTACK = 19 }; #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group" @@ -368,3 +369,8 @@ int is_terminal_support_enabled(); * Check if runC feature is enabled */ int is_runc_support_enabled(); + +/** + * Run jstack as a specific user. + */ +int run_jstack_as_user(const char *user, const char *pid, const char *jstack_path); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 1b91e8a3d6cd06..0e1b4d0e1040f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -65,6 +65,9 @@ static void display_usage(FILE *stream) { fprintf(stream, "%s container-executor --reap-runc-layer-mounts \n", de); + fprintf(stream, + " container-executor --run-jstack \n"); + fprintf(stream, " container-executor \n" " where command and command-args: \n" \ @@ -467,6 +470,16 @@ static int validate_arguments(int argc, char **argv , int *operation) { } } + if(strcmp("--run-jstack", argv[1]) == 0) { + if(argc != 5){ + fprintf(ERRORFILE, "Usage: container-executor --run-jstack \n"); + return INVALID_ARGUMENT_NUMBER; + } + + *operation = RUN_JSTACK; + return 0; + } + /* Now we have to validate 'run as user' operations that don't use a 'long option' - we should fix this at some point. The validation/argument @@ -825,6 +838,9 @@ int main(int argc, char **argv) { case REAP_RUNC_LAYER_MOUNTS: exit_code = reap_runc_layer_mounts(cmd_input.runc_layer_count); break; + case RUN_JSTACK: + exit_code = run_jstack_as_user(argv[2], argv[3], argv[4]); + break; default: fprintf(ERRORFILE, "Unexpected operation code: %d\n", operation); exit_code = INVALID_COMMAND_PROVIDED; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index f209ea53a00963..bb51a05a1f08ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -1383,6 +1383,51 @@ void test_is_empty() { } } +static int create_mock_jstack(const char *path) { + FILE *file = fopen(path, "w"); + if (file == NULL) { + return -1; + } + fprintf(file, "#!/bin/sh\necho mock-jstack\nexit 0\n"); + fclose(file); + return chmod(path, 0755); +} + +void test_run_jstack_as_user() { + char jstack_path[PATH_MAX]; + char pid_buf[32]; + pid_t child; + int status = 0; + + printf("\nTesting run_jstack_as_user\n"); + snprintf(jstack_path, sizeof(jstack_path), "%s/jstack", TEST_ROOT); + if (create_mock_jstack(jstack_path) != 0) { + printf("FAIL: could not create mock jstack at %s\n", jstack_path); + exit(1); + } + snprintf(pid_buf, sizeof(pid_buf), "%d", getpid()); + + child = fork(); + if (child == -1) { + printf("FAIL: failed to fork for run_jstack_as_user test - %s\n", + strerror(errno)); + exit(1); + } + if (child == 0) { + _exit(run_jstack_as_user(username, pid_buf, jstack_path)); + } + if (waitpid(child, &status, 0) <= 0) { + printf("FAIL: failed waiting for run_jstack_as_user child - %s\n", + strerror(errno)); + exit(1); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + printf("FAIL: run_jstack_as_user child exited with status %d\n", + WIFEXITED(status) ? WEXITSTATUS(status) : -1); + exit(1); + } +} + #define TCE_FAKE_CGROOT TEST_ROOT "/cgroup_root" #define TCE_NUM_CG_CONTROLLERS 6 extern int clean_docker_cgroups_internal(const char *mount_table, @@ -1803,6 +1848,7 @@ int main(int argc, char **argv) { test_trim_function(); test_concatenate(); + test_run_jstack_as_user(); printf("\nFinished tests\n"); printf("\nAttempting to clean up from the run\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java new file mode 100644 index 00000000000000..54af10ba927ab9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestDiagnosticJStackService.java @@ -0,0 +1,198 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.spy; + + +public class TestDiagnosticJStackService { + + private static final int NUMBER_OF_JSTACKS = 3; + private static final String DUMMY_JSTACK = + "Full thread dump OpenJDK 64-Bit Server VM (17.0.15+6-Ubuntu-0ubuntu120.04..."; + private static final String APPLICATION_ID_STR = "application_1771512066750_0001"; + private static final ApplicationId APPLICATION_ID = + ApplicationId.fromString(APPLICATION_ID_STR); + private static final String CONTAINER_ID_STR = "container_1771512066750_0001_01_000049"; + private static final ContainerId CONTAINER_ID = + ContainerId.fromString(CONTAINER_ID_STR); + private static final ApplicationACLsManager MOCK_ACLS_MANAGER = + mock(ApplicationACLsManager.class); + + private static final NodeManager.NMContext NM_CONTEXT = new NodeManager.NMContext( + null, null, null, + MOCK_ACLS_MANAGER, null, false, new Configuration() + ); + private static final DiagnosticJStackService DIAGNOSTIC_JSTACK_SERVICE = + spy(new DiagnosticJStackService(NM_CONTEXT)); + + + @Test + public void testWrongApplicationId() { + String applicationId = "app_29042"; + + assertThrows(RuntimeException.class, + () -> DIAGNOSTIC_JSTACK_SERVICE.collectApplicationThreadDump(applicationId, 3, null)); + } + + @Test + public void testCollectNodeThreadDumpSuccess() { + // No need to mock ProcessID, as it will take the unit test JVM PID + try(MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + // Wrap mockConstruction here to automatically close it + ){ + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + when(MOCK_ACLS_MANAGER.isAdmin(any())).thenReturn(true); + + String result = + DIAGNOSTIC_JSTACK_SERVICE.collectNodeThreadDump(NUMBER_OF_JSTACKS, mockRequest); + + assertEquals(NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated relative to Number of JStacks"); + + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } + + assertTrue(result.contains("--- JStack iteration 0")); + assertTrue(result.contains("--- JStack iteration 1")); + assertTrue(result.contains("--- JStack iteration 2")); + assertTrue(result.contains(DUMMY_JSTACK)); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testCollectApplicationThreadDumpSuccess() { + List pids = List.of(23L, 12L, 531L); + + Application mockApp = mock(Application.class); + NM_CONTEXT.getApplications().put(APPLICATION_ID, mockApp); + + when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); + + Map> containerPids = Map.of(CONTAINER_ID, pids); + doReturn(containerPids).when(DIAGNOSTIC_JSTACK_SERVICE).getApplicationContainerPids(mockApp); + + ProcessHandle mockProcessHandle = mock(ProcessHandle.class); + ProcessHandle.Info mockPhInfo = mock(ProcessHandle.Info.class); + + when(mockProcessHandle.info()).thenReturn(mockPhInfo); + when(mockPhInfo.user()).thenReturn(Optional.empty()); + + try(MockedStatic mockedStaticProcess = mockStatic(ProcessHandle.class); + MockedConstruction mockedConstruction = + mockConstruction(Shell.ShellCommandExecutor.class, + (mock, context) -> when(mock.getOutput()).thenReturn(DUMMY_JSTACK)) + // Wrap mockedStatic & mockedConstruction here to automatically close them + ){ + mockedStaticProcess + .when(() -> ProcessHandle.of(anyLong())).thenReturn(Optional.of(mockProcessHandle)); + + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + String result = DIAGNOSTIC_JSTACK_SERVICE + .collectApplicationThreadDump(APPLICATION_ID_STR, NUMBER_OF_JSTACKS, mockRequest); + + assertEquals(pids.size()*NUMBER_OF_JSTACKS, mockedConstruction.constructed().size(), + "ShellCommandExecutor should be instantiated for each PID time Number Of JStacks"); + + // Verify each individual mock was used exactly once + for (Shell.ShellCommandExecutor mockExecutor : mockedConstruction.constructed()) { + verify(mockExecutor, times(1)).execute(); + verify(mockExecutor, times(1)).getOutput(); + } + + assertTrue(result.contains("--- JStack iteration 0 for PID: 23 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 12 ---")); + assertTrue(result.contains("--- JStack iteration 0 for PID: 531 ---")); + assertTrue(result.contains(DUMMY_JSTACK)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + NM_CONTEXT.getApplications().remove(APPLICATION_ID); + // Clean up to avoid side effects on another test + + } + + + @Test + public void testCollectApplicationThreadDumpWhenProcessIdNotAlive() throws IOException { + int numJStacks = 3; + Application mockApp = mock(Application.class); + NM_CONTEXT.getApplications().put(APPLICATION_ID, mockApp); + + when(MOCK_ACLS_MANAGER.checkAccess(any(), any(), any(), any())).thenReturn(true); + + Map> fakeContainerPids = Map.of(CONTAINER_ID, List.of(23L)); + + doReturn(fakeContainerPids).when(DIAGNOSTIC_JSTACK_SERVICE) + .getApplicationContainerPids(mockApp); + + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + + String result = DIAGNOSTIC_JSTACK_SERVICE + .collectApplicationThreadDump(APPLICATION_ID_STR, numJStacks, mockRequest); + + assertNotNull(result); + assertTrue(result.contains("Thread Dumps for ContainerId: " + CONTAINER_ID_STR), + "Output should contain the container ID"); + assertTrue(result.contains("Status: Skipped Process with PID"), + "Since we don't mock ProcessHandle.of to return non empty it considers this PID is dead"); + + NM_CONTEXT.getApplications().remove(APPLICATION_ID); + // Clean up to avoid side effects on another test + + } + + + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md index b088c48dac70d5..75c5feb3f0904f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md @@ -787,3 +787,115 @@ Response Body: "assignedGpuDevices": [] } ``` + +JStack NodeManager API +---------------- +With JStack NodeManager API, you can get the JStack of the NodeManager processID. + +### URI + +Use the following URI to obtain resources on the NodeManager + + * http://nm-http-address:port/ws/v1/node/jstack/{numberOfJStack} + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + + None + +### GET Response Examples + +**PLAIN TEXT response** + +HTTP Request: + + GET http://nm-http-address:port/ws/v1/node/jstack/{numberOfJStack} + +Response Header: + + Cache-Control: no-cache + Pragma: no-cache + X-Content-Type-Options: nosniff + X-XSS-Protection: 1; mode=block + X-Frame-Options: SAMEORIGIN + Content-Type: text/plain + Vary: Accept-Encoding + Content-Encoding: gzip + Transfer-Encoding: chunked + + +```text +--- JStack iteration 0 for PID: 322091 --- +2026-04-23 14:39:46 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +............... +............... +JNI global refs: 66, weak refs: 0 +--- JStack iteration 1 for PID: 322091 --- +2026-04-23 14:39:46 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +``` + +JStack Application API +---------------- +With JStack Application API, you can get the JStack of all java process Containers that belongs to the ApplicationId + +### URI + +Use the following URI to obtain resources on the NodeManager + + * http://nm-http-address:port/ws/v1/node/apps/{appid}/jstack/{numberOfJStack} + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + + None + +### GET Response Examples + +**PLAIN TEXT response** + +HTTP Request: + + GET http://nm-http-address:port/ws/v1/node/apps/{appid}/jstack/{numberOfJStack} + +Response Header: + + Cache-Control: no-cache + Pragma: no-cache + X-Content-Type-Options: nosniff + X-XSS-Protection: 1; mode=block + X-Frame-Options: SAMEORIGIN + Content-Type: text/plain + Vary: Accept-Encoding + Content-Encoding: gzip + Transfer-Encoding: chunked + + +```text +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000001, PID: 358474 === +--- JStack iteration 0 for PID: 358474 --- +2026-04-23 14:54:47 +Full thread dump OpenJDK 64-Bit Server VM (17.0.11+9-adhoc.root.jdk17u mixed mode, sharing): +............... +............... +Failed to run jstack on PID: 362652 at iteration: 3 (Process likely exited before/during running jstack): Exception in thread "main" java.io.IOException: Premature EOF +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.readInt(HotSpotVirtualMachine.java:341) +at jdk.attach/sun.tools.attach.VirtualMachineImpl.execute(VirtualMachineImpl.java:197) +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.executeCommand(HotSpotVirtualMachine.java:310) +at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.remoteDataDump(HotSpotVirtualMachine.java:267) +at jdk.jcmd/sun.tools.jstack.JStack.runThreadDump(JStack.java:130) +at jdk.jcmd/sun.tools.jstack.JStack.main(JStack.java:109) + +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000038, PID: 362667 === +Failed to run jstack on PID: 362667 at iteration: 0 (Process likely exited before/during running jstack): 362667: No such process + +=== Thread Dumps for ContainerId: container_e09_1776934843524_0002_01_000039, PID: 362658 === +Status: Skipped Process with PID Process with PID 362658 is no longer exists +``` \ No newline at end of file