YARN-6302 Fail the node, if Linux Container Executor is not configured properly

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ConfigurationException.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * This exception is thrown on unrecoverable configuration errors.
+ * An example is container launch error due to configuration.
+ */
+@Public
+@Evolving
+public class ConfigurationException extends YarnException {
+
+  private static final long serialVersionUID = 0x9801a7a0f8e3L;
+
+  public ConfigurationException(Throwable cause) {
+    super(cause);
+  }
+
+  public ConfigurationException(String message) {
+    super(message);
+  }
+
+  public ConfigurationException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java

@@ -47,6 +47,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
@@ -164,9 +165,10 @@ public void prepareContainer(ContainerPrepareContext ctx) throws
    * @param ctx Encapsulates information necessary for launching containers.
    * @return the return status of the launch
    * @throws IOException if the container launch fails
+   * @throws ConfigurationException if config error was found
    */
   public abstract int launchContainer(ContainerStartContext ctx) throws
-      IOException;
+      IOException, ConfigurationException;
 
   /**
    * Signal container with the specified signal.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java

@@ -51,6 +51,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
@@ -209,7 +210,8 @@ protected ContainerLocalizer createContainerLocalizer(String user,
   }
 
   @Override
-  public int launchContainer(ContainerStartContext ctx) throws IOException {
+  public int launchContainer(ContainerStartContext ctx)
+      throws IOException, ConfigurationException {
     Container container = ctx.getContainer();
     Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
     Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
@@ -291,8 +293,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
 
       if (isContainerActive(containerId)) {
         shExec.execute();
-      }
-      else {
+      } else {
         LOG.info("Container " + containerIdStr +
             " was marked as inactive. Returning terminated error");
         return ExitCode.TERMINATED.getExitCode();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -110,6 +111,58 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   private ResourceHandler resourceHandlerChain;
   private LinuxContainerRuntime linuxContainerRuntime;
 
+  /**
+   * The container exit code.
+   */
+  public enum ExitCode {
+    SUCCESS(0),
+    INVALID_ARGUMENT_NUMBER(1),
+    INVALID_COMMAND_PROVIDED(3),
+    INVALID_NM_ROOT_DIRS(5),
+    SETUID_OPER_FAILED(6),
+    UNABLE_TO_EXECUTE_CONTAINER_SCRIPT(7),
+    UNABLE_TO_SIGNAL_CONTAINER(8),
+    INVALID_CONTAINER_PID(9),
+    OUT_OF_MEMORY(18),
+    INITIALIZE_USER_FAILED(20),
+    PATH_TO_DELETE_IS_NULL(21),
+    INVALID_CONTAINER_EXEC_PERMISSIONS(22),
+    INVALID_CONFIG_FILE(24),
+    SETSID_OPER_FAILED(25),
+    WRITE_PIDFILE_FAILED(26),
+    WRITE_CGROUP_FAILED(27),
+    TRAFFIC_CONTROL_EXECUTION_FAILED(28),
+    DOCKER_RUN_FAILED(29),
+    ERROR_OPENING_DOCKER_FILE(30),
+    ERROR_READING_DOCKER_FILE(31),
+    FEATURE_DISABLED(32),
+    COULD_NOT_CREATE_SCRIPT_COPY(33),
+    COULD_NOT_CREATE_CREDENTIALS_FILE(34),
+    COULD_NOT_CREATE_WORK_DIRECTORIES(35),
+    COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36),
+    COULD_NOT_CREATE_TMP_DIRECTORIES(37),
+    ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38);
+
+    private final int code;
+
+    ExitCode(int exitCode) {
+      this.code = exitCode;
+    }
+
+    /**
+     * Get the exit code as an int.
+     * @return the exit code as an int
+     */
+    public int getExitCode() {
+      return code;
+    }
+
+    @Override
+    public String toString() {
+      return String.valueOf(code);
+    }
+  }
+
   /**
    * Default constructor to allow for creation through reflection.
    */
@@ -386,7 +439,8 @@ public void prepareContainer(ContainerPrepareContext ctx) throws IOException {
   }
 
   @Override
-  public int launchContainer(ContainerStartContext ctx) throws IOException {
+  public int launchContainer(ContainerStartContext ctx)
+      throws IOException, ConfigurationException {
     Container container = ctx.getContainer();
     Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
     Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
@@ -496,16 +550,16 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
       } else {
         LOG.info(
             "Container was marked as inactive. Returning terminated error");
-        return ExitCode.TERMINATED.getExitCode();
+        return ContainerExecutor.ExitCode.TERMINATED.getExitCode();
       }
     } catch (ContainerExecutionException e) {
       int exitCode = e.getExitCode();
       LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
       // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
       // terminated/killed forcefully. In all other cases, log the
       // output
-      if (exitCode != ExitCode.FORCE_KILLED.getExitCode()
-          && exitCode != ExitCode.TERMINATED.getExitCode()) {
+      if (exitCode != ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode()
+          && exitCode != ContainerExecutor.ExitCode.TERMINATED.getExitCode()) {
         LOG.warn("Exception from container-launch with container ID: "
             + containerId + " and exit code: " + exitCode, e);
 
@@ -525,6 +579,23 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
         logOutput(diagnostics);
         container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
             diagnostics));
+        if (exitCode ==
+                ExitCode.INVALID_CONTAINER_EXEC_PERMISSIONS.getExitCode() ||
+            exitCode ==
+                ExitCode.INVALID_CONFIG_FILE.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_SCRIPT_COPY.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_CREDENTIALS_FILE.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_WORK_DIRECTORIES.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_APP_LOG_DIRECTORIES.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_TMP_DIRECTORIES.getExitCode()) {
+          throw new ConfigurationException(
+              "Linux Container Executor reached unrecoverable exception", e);
+        }
       } else {
         container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
             "Container killed on request. Exit code is " + exitCode));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeHealthCheckerService.java

@@ -18,10 +18,14 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import com.google.common.base.Joiner;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.NodeHealthScriptRunner;
 
+import java.util.Arrays;
+import java.util.Collections;
+
 /**
  * The class which provides functionality of checking the health of the node and
  * reporting back to the service for which the health checker has been asked to
@@ -31,6 +35,8 @@ public class NodeHealthCheckerService extends CompositeService {
 
   private NodeHealthScriptRunner nodeHealthScriptRunner;
   private LocalDirsHandlerService dirsHandler;
+  private Exception nodeHealthException;
+  private long nodeHealthExceptionReportTime;
 
   static final String SEPARATOR = ";";
 
@@ -39,6 +45,8 @@ public NodeHealthCheckerService(NodeHealthScriptRunner scriptRunner,
     super(NodeHealthCheckerService.class.getName());
     nodeHealthScriptRunner = scriptRunner;
     dirsHandler = dirHandlerService;
+    nodeHealthException = null;
+    nodeHealthExceptionReportTime = 0;
   }
 
   @Override
@@ -50,37 +58,48 @@ protected void serviceInit(Configuration conf) throws Exception {
     super.serviceInit(conf);
   }
 
+  private String nullIfEmpty(String in) {
+    if (in == null || in.equals("")) {
+      return null;
+    } else {
+      return in;
+    }
+  }
+
   /**
    * @return the reporting string of health of the node
    */
   String getHealthReport() {
-    String scriptReport = (nodeHealthScriptRunner == null) ? ""
-        : nodeHealthScriptRunner.getHealthReport();
-    if (scriptReport.equals("")) {
-      return dirsHandler.getDisksHealthReport(false);
-    } else {
-      return scriptReport.concat(SEPARATOR + dirsHandler.getDisksHealthReport(false));
-    }
+    String scriptReport = nullIfEmpty(nodeHealthScriptRunner == null ? null :
+        nodeHealthScriptRunner.getHealthReport());
+    String discReport =
+        nullIfEmpty(dirsHandler.getDisksHealthReport(false));
+    String exceptionReport = nullIfEmpty(nodeHealthException == null ? null :
+        nodeHealthException.getMessage());
+
+    return Joiner.on(SEPARATOR).skipNulls()
+        .join(scriptReport, discReport, exceptionReport);
   }
 
   /**
    * @return <em>true</em> if the node is healthy
    */
   boolean isHealthy() {
-    boolean scriptHealthStatus = (nodeHealthScriptRunner == null) ? true
-        : nodeHealthScriptRunner.isHealthy();
-    return scriptHealthStatus && dirsHandler.areDisksHealthy();
+    boolean scriptHealthy = nodeHealthScriptRunner == null ||
+        nodeHealthScriptRunner.isHealthy();
+    return nodeHealthException == null &&
+        scriptHealthy && dirsHandler.areDisksHealthy();
   }
 
   /**
    * @return when the last time the node health status is reported
    */
   long getLastHealthReportTime() {
-    long diskCheckTime = dirsHandler.getLastDisksCheckTime();
-    long lastReportTime = (nodeHealthScriptRunner == null)
-        ? diskCheckTime
-        : Math.max(nodeHealthScriptRunner.getLastReportedTime(), diskCheckTime);
-    return lastReportTime;
+    return Collections.max(Arrays.asList(
+        dirsHandler.getLastDisksCheckTime(),
+        nodeHealthScriptRunner == null ? 0 :
+            nodeHealthScriptRunner.getLastReportedTime(),
+        nodeHealthExceptionReportTime));
   }
 
   /**
@@ -96,4 +115,13 @@ public LocalDirsHandlerService getDiskHandler() {
   NodeHealthScriptRunner getNodeHealthScriptRunner() {
     return nodeHealthScriptRunner;
   }
+
+  /**
+   * Report an exception to mark the node as unhealthy.
+   * @param ex the exception that makes the node unhealthy
+   */
+  void reportException(Exception ex) {
+    nodeHealthException = ex;
+    nodeHealthExceptionReportTime = System.currentTimeMillis();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java

@@ -53,4 +53,10 @@ public interface NodeStatusUpdater extends Service {
    * Clear the list of recently completed containers
    */
   public void clearFinishedContainersFromCache();
+
+  /**
+   * Report an unrecoverable exception.
+   * @param ex exception that makes the node unhealthy
+   */
+  void reportException(Exception ex);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java

@@ -990,6 +990,12 @@ private boolean handleShutdownOrResyncCommand(
     return false;
   }
 
+  @Override
+  public void reportException(Exception ex) {
+    healthChecker.reportException(ex);
+    sendOutofBandHeartBeat();
+  }
+
   private List<LogAggregationReport> getLogAggregationReportsForApps(
       ConcurrentLinkedQueue<LogAggregationReport> lastestLogAggregationStatus) {
     LogAggregationReport status;