From 4eb3590dfa90e39c1275c39559b891f5eb0968a5 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Mon, 28 Sep 2026 17:40:12 +0200 Subject: [PATCH 01/15] Add dummy status command --- cardano-testnet/README.md | 3 +- cardano-testnet/cardano-testnet.cabal | 3 ++ cardano-testnet/src/Parsers/Run.hs | 6 +++ cardano-testnet/src/Testnet/Status/Parsers.hs | 50 +++++++++++++++++++ cardano-testnet/src/Testnet/Status/Run.hs | 10 ++++ cardano-testnet/src/Testnet/Status/Types.hs | 21 ++++++++ .../files/golden/help.cli | 9 +++- .../files/golden/help/status.cli | 14 ++++++ 8 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 cardano-testnet/src/Testnet/Status/Parsers.hs create mode 100644 cardano-testnet/src/Testnet/Status/Run.hs create mode 100644 cardano-testnet/src/Testnet/Status/Types.hs create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index f9fb0cd0e60..28f03b73347 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -53,7 +53,7 @@ export CARDANO_NODE=path to your executable --> ```text -Usage: cardano-testnet (cardano | create-env | version | help) +Usage: cardano-testnet (cardano | create-env | status | version | help) Available options: -h,--help Show this help text @@ -61,6 +61,7 @@ Available options: Available commands: cardano Start a testnet and keep it running until stopped create-env Create a sandbox for Cardano testnet + status Check the status of a running testnet version Show cardano-testnet version help Show cardano-testnet help ``` diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 94f346fddea..6a115657752 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -132,6 +132,9 @@ library Testnet.Start.Byron Testnet.Start.Cardano Testnet.Start.Types + Testnet.Status.Parsers + Testnet.Status.Run + Testnet.Status.Types Testnet.SubmitApi Testnet.TestQueryCmds Testnet.Types diff --git a/cardano-testnet/src/Parsers/Run.hs b/cardano-testnet/src/Parsers/Run.hs index d3d2a9ed27d..acc66b0b80b 100644 --- a/cardano-testnet/src/Parsers/Run.hs +++ b/cardano-testnet/src/Parsers/Run.hs @@ -27,6 +27,9 @@ import System.Directory (doesDirectoryExist) import Testnet.Filepath (unTmpAbsPath) import Testnet.Start.Cardano import Testnet.Start.Types +import Testnet.Status.Parsers (cmdCheckStatus) +import Testnet.Status.Run (runCheckStatusOptions) +import Testnet.Status.Types (CheckStatusOptions) import Testnet.Types (TestnetNode (..)) import Parsers.Cardano @@ -62,6 +65,7 @@ opts envCli = Opt.info (commands envCli <**> helper) idm data CardanoTestnetCommands = StartCardanoTestnet CardanoTestnetCliOptions | CreateTestnetEnv CardanoTestnetCreateEnvOptions + | CheckStatus CheckStatusOptions | GetVersion VersionOptions | Help ParserPrefs (ParserInfo CardanoTestnetCommands) HelpOptions @@ -70,6 +74,7 @@ commands envCli = asum [ fmap StartCardanoTestnet (subparser cmdCardano) , fmap CreateTestnetEnv (subparser cmdCreateEnv) + , fmap CheckStatus (subparser cmdCheckStatus) , fmap GetVersion (subparser cmdVersion) , fmap (Help pref (opts envCli)) (subparser cmdHelp) ] @@ -79,6 +84,7 @@ runTestnetCmd :: CardanoTestnetCommands -> IO () runTestnetCmd = \case StartCardanoTestnet cmdOpts -> runCardanoOptions cmdOpts CreateTestnetEnv cmdOpts -> createEnvOptions cmdOpts + CheckStatus cmdOpts -> runCheckStatusOptions cmdOpts GetVersion cmdOpts -> runVersionOptions cmdOpts Help pPrefs pInfo cmdOpts -> runHelpOptions pPrefs pInfo cmdOpts diff --git a/cardano-testnet/src/Testnet/Status/Parsers.hs b/cardano-testnet/src/Testnet/Status/Parsers.hs new file mode 100644 index 00000000000..fafa8e30af0 --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Parsers.hs @@ -0,0 +1,50 @@ +module Testnet.Status.Parsers + ( cmdCheckStatus + ) where + +import Cardano.CLI.EraBased.Common.Option (command') +import Cardano.Prelude (Nat) + +import Prelude + +import Control.Monad (when) +import Data.Default.Class (def) +import Options.Applicative (CommandFields, Mod, Parser) +import qualified Options.Applicative as OA + +import Testnet.Status.Types (CheckStatusOptions (..), OutputFormat (..)) + +cmdCheckStatus :: Mod CommandFields CheckStatusOptions +cmdCheckStatus = command' "status" "Check the status of a running testnet" optsCheckStatus + +optsCheckStatus :: Parser CheckStatusOptions +optsCheckStatus = CheckStatusOptions + <$> pOutputDir + <*> pStatusCheckTimeout + <*> pOutputFormat + +pOutputDir :: Parser (Maybe FilePath) +pOutputDir = OA.optional (OA.strOption + ( OA.long "testnet-dir" + <> OA.metavar "FILEPATH" + <> OA.help "Path to the environment folder of the testnet whose status to check, by default \"./testnet\"." + )) + + +pStatusCheckTimeout :: Parser Nat +pStatusCheckTimeout = OA.option timeoutReader + ( OA.long "timeout" + <> OA.metavar "SECONDS" + <> OA.help "How many seconds to wait for each node to answer (1 - 3600)." + <> OA.value 5 + <> OA.showDefault + ) + where + timeoutReader = OA.auto >>= \n -> do + when (n < 1 || n > 3600) $ + OA.readerError $ "Timeout out of range (1 - 3600): " <> show n + pure n + +pOutputFormat :: Parser OutputFormat +pOutputFormat = + OA.flag def OutputFormatJson (OA.long "json" <> OA.help "Output status as JSON instead of a table.") diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs new file mode 100644 index 00000000000..e863a79dbe2 --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -0,0 +1,10 @@ +module Testnet.Status.Run + ( runCheckStatusOptions + ) where + +import Prelude + +import Testnet.Status.Types (CheckStatusOptions) + +runCheckStatusOptions :: CheckStatusOptions -> IO () +runCheckStatusOptions _ = pure () diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs new file mode 100644 index 00000000000..c5d0920a2f6 --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -0,0 +1,21 @@ +module Testnet.Status.Types + ( CheckStatusOptions (..) + , OutputFormat(..) + ) where + +import Cardano.Prelude (Nat) + +import Prelude + +import Data.Default.Class (Default (..)) + +data CheckStatusOptions = CheckStatusOptions + { testnetDir :: Maybe FilePath + , statusCheckTimeout :: Nat + , outputFormat :: OutputFormat + } + +data OutputFormat = OutputFormatText | OutputFormatJson + +instance Default OutputFormat where + def = OutputFormatText diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli index 12a97b9e81b..1a34d1930e0 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli @@ -1,4 +1,4 @@ -Usage: cardano-testnet (cardano | create-env | version | help) +Usage: cardano-testnet (cardano | create-env | status | version | help) Usage: cardano-testnet cardano [ --node-env FILEPATH [--preserve-timestamps] @@ -39,6 +39,13 @@ Usage: cardano-testnet create-env Create a sandbox for Cardano testnet +Usage: cardano-testnet status [--testnet-dir FILEPATH] + [--timeout SECONDS] + [--json] + + + Check the status of a running testnet + Usage: cardano-testnet version Show cardano-testnet version diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli new file mode 100644 index 00000000000..b8a19886b61 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli @@ -0,0 +1,14 @@ +Usage: cardano-testnet status [--testnet-dir FILEPATH] + [--timeout SECONDS] + [--json] + + + Check the status of a running testnet + +Available options: + --testnet-dir FILEPATH Path to the environment folder of the testnet whose + status to check, by default "./testnet". + --timeout SECONDS How many seconds to wait for each node to answer (1 - + 3600). (default: 5) + --json Output status as JSON instead of a table. + -h,--help Show this help text From 40de240174a1ef3abd4b7c3726229a33625016ad Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 29 Sep 2026 00:36:28 +0200 Subject: [PATCH 02/15] Implement table printing and define types for status command --- cardano-testnet/src/Testnet/Status/Run.hs | 19 +++++- cardano-testnet/src/Testnet/Status/Types.hs | 64 ++++++++++++++++++++- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index e863a79dbe2..aaf38ce56e0 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -2,9 +2,26 @@ module Testnet.Status.Run ( runCheckStatusOptions ) where +import Cardano.Prelude (forM_, transpose) + import Prelude import Testnet.Status.Types (CheckStatusOptions) runCheckStatusOptions :: CheckStatusOptions -> IO () -runCheckStatusOptions _ = pure () +runCheckStatusOptions _ = + printTable [] + + +printTable :: [[String]] -> IO () +printTable rows = do + forM_ rows $ \row -> do + putStrLn $ concat ("+":["-" ++ replicate width '-' ++ "-+" | width <- widths]) + putStrLn $ concat ("|":[ " " ++ pad width cell ++ " |" | (cell, width) <- zip row widths]) + putStrLn $ concat ("+":["-" ++replicate width '-' ++ "-+" | width <- widths]) + where + widths :: [Int] + widths = map (maximum . map length) (transpose rows) + + pad :: Int -> String -> [Char] + pad w s = s ++ replicate (w - length s) ' ' diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index c5d0920a2f6..bd223cb7c45 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -1,21 +1,79 @@ +{-# LANGUAGE InstanceSigs #-} module Testnet.Status.Types ( CheckStatusOptions (..) , OutputFormat(..) ) where +import Cardano.Api (BlockHeader, BlockNo, Hash, SlotNo) + import Cardano.Prelude (Nat) import Prelude import Data.Default.Class (Default (..)) +import Data.Time (NominalDiffTime, UTCTime) +import System.Process (Pid) + +import Testnet.Manifest (ManifestNodeRole) + +-- Option types +-- | Parameters for the `status` subcommand data CheckStatusOptions = CheckStatusOptions - { testnetDir :: Maybe FilePath - , statusCheckTimeout :: Nat - , outputFormat :: OutputFormat + { testnetDir :: Maybe FilePath -- ^ the directory with the testnet environment, by default "./testnet" + , statusCheckTimeout :: Nat -- ^ the timeout for waiting for each node to respond to the status check in seconds + , outputFormat :: OutputFormat -- ^ the format in which to output the status information } +-- | Output format for the `status` subcommand data OutputFormat = OutputFormatText | OutputFormatJson instance Default OutputFormat where + def :: OutputFormat def = OutputFormatText + +-- Output types + +-- | The state of the whole network. +data NetworkStatus + = NetworkRunning -- ^ every node is 'NodeOk' + | NetworkDegraded -- ^ some nodes are 'NodeOk', but not all + | NetworkStalled -- ^ some nodes answer, but none is 'NodeOk' + | NetworkStopped -- ^ no node answers + | NoTestnet -- ^ there is no manifest file in the 'testnetDir' + deriving (Eq, Show) + +-- | The state of one node. +data NodeState + = NodeOk -- ^ it answers, and its tip is fresh + | NodeStalled -- ^ it answers, but its tip is too old, or it has no blocks yet + | NodeUnreachable -- ^ it does not answer, but process may be alive + | NodeDown -- ^ it does not answer and the process is gone + deriving (Eq, Show) + +-- | A node's chain tip. +data TipInfo = TipInfo + { tipSlot :: !SlotNo + , tipBlockNo :: !BlockNo + , tipHash :: Hash BlockHeader -- ^ hex + , tipAge :: !NominalDiffTime -- ^ time since the start of the tip's slot + } deriving (Eq, Show) + +-- | What 'checkStatus' found out about one node. +data NodeProbeResult = NodeProbeResult + { probeNodeName :: !String + , probeNodeRole :: !ManifestNodeRole + , probePid :: !(Maybe Pid) + , probePidIsAlive :: !(Maybe Bool) -- ^ 'Nothing' when unknown: no pid, or Windows + , probeNodeState :: !NodeState + , probeNodeTipInfo :: !(Maybe TipInfo) -- ^ 'Just' when the node answered with a tip + } deriving (Eq, Show) + +data StatusReport = StatusReport + { reportOutputDir :: !FilePath + , reportCheckedAt :: !UTCTime + , reportStatus :: !NetworkStatus + , reportBestTip :: !(Maybe TipInfo) -- ^ the most tip of all nodes + , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' + , reportProblemExplanation :: !(Maybe String) -- ^ why there is no testnet + } deriving (Eq, Show) From 509f4838c8631d9960134920c83b9c6f09a9ee23 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 29 Sep 2026 14:33:43 +0200 Subject: [PATCH 03/15] Add `ToJSON` instances to output types --- cardano-testnet/src/Testnet/Status/Types.hs | 72 ++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index bd223cb7c45..0d9c083f8e6 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -1,12 +1,14 @@ {-# LANGUAGE InstanceSigs #-} +{-# LANGUAGE OverloadedStrings #-} + module Testnet.Status.Types ( CheckStatusOptions (..) , OutputFormat(..) ) where -import Cardano.Api (BlockHeader, BlockNo, Hash, SlotNo) +import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), ToJSON (..)) -import Cardano.Prelude (Nat) +import Cardano.Prelude (Nat, ExitCode (..)) import Prelude @@ -15,6 +17,7 @@ import Data.Time (NominalDiffTime, UTCTime) import System.Process (Pid) import Testnet.Manifest (ManifestNodeRole) +import Data.Aeson (Value, object, (.=)) -- Option types @@ -34,6 +37,11 @@ instance Default OutputFormat where -- Output types +-- | Version of the JSON output, pinned by +-- @cardano-testnet/schemas/status.schema.json@. +statusSchemaVersion :: Int +statusSchemaVersion = 1 + -- | The state of the whole network. data NetworkStatus = NetworkRunning -- ^ every node is 'NodeOk' @@ -43,6 +51,14 @@ data NetworkStatus | NoTestnet -- ^ there is no manifest file in the 'testnetDir' deriving (Eq, Show) +instance ToJSON NetworkStatus where + toJSON :: NetworkStatus -> Value + toJSON NetworkRunning = "running" + toJSON NetworkDegraded = "degraded" + toJSON NetworkStalled = "stalled" + toJSON NetworkStopped = "stopped" + toJSON NoTestnet = "no-testnet" + -- | The state of one node. data NodeState = NodeOk -- ^ it answers, and its tip is fresh @@ -51,6 +67,13 @@ data NodeState | NodeDown -- ^ it does not answer and the process is gone deriving (Eq, Show) +instance ToJSON NodeState where + toJSON :: NodeState -> Value + toJSON NodeOk = "ok" + toJSON NodeStalled = "stalled" + toJSON NodeUnreachable = "unreachable" + toJSON NodeDown = "down" + -- | A node's chain tip. data TipInfo = TipInfo { tipSlot :: !SlotNo @@ -59,6 +82,15 @@ data TipInfo = TipInfo , tipAge :: !NominalDiffTime -- ^ time since the start of the tip's slot } deriving (Eq, Show) +instance ToJSON TipInfo where + toJSON :: TipInfo -> Value + toJSON t = object + [ "slot" .= unSlotNo (tipSlot t) + , "blockNo" .= unBlockNo (tipBlockNo t) + , "hash" .= tipHash t + , "ageSeconds" .= tipAge t + ] + -- | What 'checkStatus' found out about one node. data NodeProbeResult = NodeProbeResult { probeNodeName :: !String @@ -69,6 +101,17 @@ data NodeProbeResult = NodeProbeResult , probeNodeTipInfo :: !(Maybe TipInfo) -- ^ 'Just' when the node answered with a tip } deriving (Eq, Show) +instance ToJSON NodeProbeResult where + toJSON :: NodeProbeResult -> Value + toJSON n = object + [ "name" .= probeNodeName n + , "role" .= probeNodeRole n + , "pid" .= (fromIntegral <$> probePid n :: Maybe Int) + , "pidAlive" .= probePidIsAlive n + , "state" .= probeNodeState n + , "tip" .= probeNodeTipInfo n + ] + data StatusReport = StatusReport { reportOutputDir :: !FilePath , reportCheckedAt :: !UTCTime @@ -77,3 +120,28 @@ data StatusReport = StatusReport , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' , reportProblemExplanation :: !(Maybe String) -- ^ why there is no testnet } deriving (Eq, Show) + +instance ToJSON StatusReport where + toJSON :: StatusReport -> Value + toJSON r = object + [ "schemaVersion" .= statusSchemaVersion + , "checkedAt" .= reportCheckedAt r + , "status" .= reportStatus r + , "exitCode" .= exitCodeNumber (exitCodeForStatus (reportStatus r)) + , "chainProducingBlocks" .= (exitCodeForStatus (reportStatus r) == ExitSuccess) + , "bestTip" .= reportBestTip r + , "nodes" .= reportNodes r + ] + where + -- Extract code number from 'ExitCode' + exitCodeNumber :: ExitCode -> Int + exitCodeNumber ExitSuccess = 0 + exitCodeNumber (ExitFailure n) = n + + -- | Convert the network state to an exit code + exitCodeForStatus :: NetworkStatus -> ExitCode + exitCodeForStatus NetworkRunning = ExitSuccess + exitCodeForStatus NetworkDegraded = ExitSuccess + exitCodeForStatus NetworkStalled = ExitFailure 3 + exitCodeForStatus NetworkStopped = ExitFailure 4 + exitCodeForStatus NoTestnet = ExitFailure 5 \ No newline at end of file From a225612db311cdc05d2a0f0e0ab5241863e0664a Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Thu, 8 Oct 2026 21:17:52 +0200 Subject: [PATCH 04/15] Implement human-readable rendering of `StatusReport` --- cardano-testnet/src/Testnet/Status/Run.hs | 136 ++++++++++++++++++-- cardano-testnet/src/Testnet/Status/Types.hs | 11 +- 2 files changed, 131 insertions(+), 16 deletions(-) diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index aaf38ce56e0..12434695bac 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -1,24 +1,134 @@ -module Testnet.Status.Run - ( runCheckStatusOptions - ) where +module Testnet.Status.Run ( + runCheckStatusOptions, +) where -import Cardano.Prelude (forM_, transpose) +import Cardano.Api (BlockNo (..), SlotNo (..)) + +import Cardano.Node.Testnet.Paths (defaultManifestFile) +import Cardano.Prelude (transpose) import Prelude -import Testnet.Status.Types (CheckStatusOptions) +import Data.Time (NominalDiffTime, getCurrentTime) +import Text.Printf (printf) + +import Testnet.Manifest (ManifestNodeRole (..)) +import Testnet.Status.Types (CheckStatusOptions, NetworkStatus (..), NodeProbeResult (..), + NodeState (..), StatusReport (..), TipInfo (..)) runCheckStatusOptions :: CheckStatusOptions -> IO () -runCheckStatusOptions _ = - printTable [] +runCheckStatusOptions _ = do + currentTime <- getCurrentTime + putStr $ + renderStatusReportAsTable + ( StatusReport + { reportOutputDir = "./testnet" + , reportCheckedAt = currentTime + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just "Not implemented yet" + } + ) + +renderStatusReportAsTable :: StatusReport -> String +renderStatusReportAsTable + StatusReport + { reportStatus = status + , reportBestTip = bestTip + , reportNodes = nodes + , reportOutputDir = outputDir + , reportProblemExplanation = problemExplanation + } = + unlines $ ("Network: " <> summary status) : details + where + notOk = length $ filter ((/= NodeOk) . probeNodeState) nodes + + seconds :: NominalDiffTime -> String + seconds s = printf "%.1fs" (realToFrac s :: Double) + + summary :: NetworkStatus -> String + summary NetworkRunning = + "RUNNING - chain is producing blocks" + <> foldMap + ( \TipInfo + { tipSlot = slotNo + , tipBlockNo = blockNo + , tipAge = age + } -> + " (tip: slot " + <> show (unSlotNo slotNo) + <> ", block " + <> show (unBlockNo blockNo) + <> ", " + <> seconds age + <> " old)" + ) + bestTip + summary NetworkDegraded = + "DEGRADED - chain is producing blocks, but " + <> show notOk + <> " of " + <> show (length nodes) + <> (if notOk == 1 then " nodes is" else " nodes are") + <> " not ok" + summary NetworkStalled = + "STALLED - chain is not producing blocks" <> case bestTip of + Just t -> " (tip is " <> seconds (tipAge t) <> " old)" + _ -> " (no node has a block yet)" + summary NetworkStopped = "STOPPED - no node answers" + summary NoTestnet = "NO-TESTNET - no " <> defaultManifestFile <> " in " <> outputDir + details :: [String] + details = case status of + NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> x <> ")") problemExplanation] + _ -> + renderTable $ + ["NODE", "ROLE", "STATE", "PID", "TIP SLOT", "TIP BLOCK", "TIP AGE"] + : [ [ nodeName + , case nodeRole of + RoleSpo -> "spo" + RoleRelay -> "relay" + , case nodeState of + NodeOk -> "ok" + NodeStalled -> "stalled" + NodeUnreachable -> "unreachable" + NodeDown -> "down" + , maybe "-" show mNodePid + ] + <> case mNodeTipInfo of + Nothing -> ["-", "-", "-"] + Just + ( TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipAge = age + } + ) -> + [ show $ unSlotNo slot + , show $ unBlockNo blockNo + , seconds age + ] + | NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = nodeRole + , probePid = mNodePid + , probeNodeState = nodeState + , probeNodeTipInfo = mNodeTipInfo + } <- + nodes + ] -printTable :: [[String]] -> IO () -printTable rows = do - forM_ rows $ \row -> do - putStrLn $ concat ("+":["-" ++ replicate width '-' ++ "-+" | width <- widths]) - putStrLn $ concat ("|":[ " " ++ pad width cell ++ " |" | (cell, width) <- zip row widths]) - putStrLn $ concat ("+":["-" ++replicate width '-' ++ "-+" | width <- widths]) +renderTable :: [[String]] -> [String] +renderTable rows = + ("+" ++ concat ["-" ++ replicate width '-' ++ "-+" | width <- widths]) + : concatMap + ( \row -> + [ concat ("|" : [" " ++ pad width cell ++ " |" | (cell, width) <- zip row widths]) + , concat ("+" : ["-" ++ replicate width '-' ++ "-+" | width <- widths]) + ] + ) + rows where widths :: [Int] widths = map (maximum . map length) (transpose rows) diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index 0d9c083f8e6..7bd98e53c86 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -3,21 +3,26 @@ module Testnet.Status.Types ( CheckStatusOptions (..) + , NetworkStatus(..) + , NodeProbeResult(..) + , NodeState(..) , OutputFormat(..) + , StatusReport(..) + , TipInfo(..) ) where import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), ToJSON (..)) -import Cardano.Prelude (Nat, ExitCode (..)) +import Cardano.Prelude (ExitCode (..), Nat) import Prelude +import Data.Aeson (Value, object, (.=)) import Data.Default.Class (Default (..)) import Data.Time (NominalDiffTime, UTCTime) import System.Process (Pid) import Testnet.Manifest (ManifestNodeRole) -import Data.Aeson (Value, object, (.=)) -- Option types @@ -144,4 +149,4 @@ instance ToJSON StatusReport where exitCodeForStatus NetworkDegraded = ExitSuccess exitCodeForStatus NetworkStalled = ExitFailure 3 exitCodeForStatus NetworkStopped = ExitFailure 4 - exitCodeForStatus NoTestnet = ExitFailure 5 \ No newline at end of file + exitCodeForStatus NoTestnet = ExitFailure 5 From c8a7b893980878a4313663343613f6cf240b1379 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Fri, 9 Oct 2026 22:03:42 +0200 Subject: [PATCH 05/15] Implement status command --- cardano-testnet/src/Testnet/Signal.hs | 25 +- cardano-testnet/src/Testnet/Status/Run.hs | 240 ++++++++++++++++++-- cardano-testnet/src/Testnet/Status/Types.hs | 26 ++- cardano-testnet/src/Testnet/Types.hs | 8 +- 4 files changed, 265 insertions(+), 34 deletions(-) diff --git a/cardano-testnet/src/Testnet/Signal.hs b/cardano-testnet/src/Testnet/Signal.hs index 5523da03993..a3fc06b86af 100644 --- a/cardano-testnet/src/Testnet/Signal.hs +++ b/cardano-testnet/src/Testnet/Signal.hs @@ -10,15 +10,17 @@ module Testnet.Signal ( hardKillProcess , interruptNodesOnSigINT + , isProcessAlive ) where #ifdef UNIX import Control.Monad -import System.Posix.Signals (Handler (..), installHandler, raiseSignal, sigINT, - sigKILL, signalProcess) -import System.Process (ProcessHandle, getPid, interruptProcessGroupOf) +import System.IO.Error (isDoesNotExistError, tryIOError) +import System.Posix.Signals (Handler (..), installHandler, nullSignal, raiseSignal, + sigINT, sigKILL, signalProcess) +import System.Process (Pid, ProcessHandle, getPid, interruptProcessGroupOf) #else -import System.Process (ProcessHandle, terminateProcess) +import System.Process (Pid, ProcessHandle, terminateProcess) #endif import Data.List.NonEmpty (NonEmpty) @@ -49,3 +51,18 @@ hardKillProcess hProcess = getPid hProcess >>= mapM_ (signalProcess sigKILL) #else hardKillProcess = terminateProcess #endif + +-- | Whether a process with the given pid exists: 'Just' the answer on unix, +-- where signal 0 checks this without sending anything, and 'Nothing' on +-- Windows. A dead process that nobody has waited for yet (a zombie) still +-- exists, and so does an unrelated process that reuses the pid. +isProcessAlive :: Pid -> IO (Maybe Bool) +#ifdef UNIX +isProcessAlive pid = do + result <- tryIOError $ signalProcess nullSignal pid + pure . Just $ case result of + Left e | isDoesNotExistError e -> False -- no such process + _ -> True -- signalled, or not allowed to: either way it exists +#else +isProcessAlive _ = pure Nothing +#endif diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 12434695bac..59fd67ae174 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -1,38 +1,235 @@ +{-# LANGUAGE NumericUnderscores #-} + module Testnet.Status.Run ( runCheckStatusOptions, ) where -import Cardano.Api (BlockNo (..), SlotNo (..)) +import Cardano.Api (BlockNo (..), CardanoEra (..), ChainTip (..), + ConsensusModeParams (..), LocalNodeConnectInfo (..), NetworkId (..), + NetworkMagic (..), ShelleyGenesis, SlotNo (..), getLocalChainTip, mapFile) -import Cardano.Node.Testnet.Paths (defaultManifestFile) -import Cardano.Prelude (transpose) +import qualified Cardano.Ledger.Shelley.API as SL +import Cardano.Node.Testnet.Paths (defaultGenesisFilepath, defaultManifestFile) +import Cardano.Prelude (ExceptT (..), ExitCode (..), MonadIO (..), Nat, NonEmpty, + SomeException, Word32, runExceptT, toList, transpose, try) import Prelude -import Data.Time (NominalDiffTime, getCurrentTime) +import Data.Aeson (FromJSON, eitherDecodeFileStrict') +import Data.Aeson.Encode.Pretty (encodePretty) +import qualified Data.ByteString.Lazy as LBS +import Data.Either (fromRight) +import Data.Either.Extra (mapLeft) +import qualified Data.List as List +import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) +import Data.Time (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime) +import System.Directory (doesDirectoryExist, doesFileExist) +import System.Exit (exitWith) +import System.FilePath (()) +import qualified System.IO as IO +import System.IO.Error (ioeGetErrorString, tryIOError) +import System.Timeout (timeout) import Text.Printf (printf) -import Testnet.Manifest (ManifestNodeRole (..)) -import Testnet.Status.Types (CheckStatusOptions, NetworkStatus (..), NodeProbeResult (..), - NodeState (..), StatusReport (..), TipInfo (..)) +import Testnet.ChainWatchdog (chainForecastHorizon, chainStallTimeoutFromHorizon) +import Testnet.Manifest (Manifest (..), ManifestGenesisFiles (..), ManifestNetwork (..), + ManifestNode (..), ManifestNodeRole (..), ManifestPaths (..)) +import Testnet.Signal (isProcessAlive) +import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (..), NodeAnswer (..), + NodeProbeResult (..), NodeState (..), OutputFormat (..), StatusReport (..), + TipInfo (..), exitCodeForStatus) +import Testnet.Types (sgSlotLength, sgSystemStart, testnetEpochSlots) + +import UnliftIO.Async (forConcurrently) runCheckStatusOptions :: CheckStatusOptions -> IO () -runCheckStatusOptions _ = do - currentTime <- getCurrentTime - putStr $ - renderStatusReportAsTable - ( StatusReport - { reportOutputDir = "./testnet" - , reportCheckedAt = currentTime - , reportStatus = NoTestnet - , reportBestTip = Nothing - , reportNodes = [] - , reportProblemExplanation = Just "Not implemented yet" +runCheckStatusOptions + CheckStatusOptions + { testnetDir = dir + , statusCheckTimeout = timeoutSeconds + , outputFormat = format + } = do + eStatus <- checkStatus (fromMaybe "./testnet" dir) timeoutSeconds + status <- + either + ( \err -> do + IO.hPutStrLn IO.stderr $ "cardano-testnet status: " <> err + exitWith $ ExitFailure 1 + ) + pure + eStatus + case format of + OutputFormatText -> putStr $ renderStatusReportHumanReadable status + OutputFormatJson -> do LBS.putStr $ encodePretty status + putStrLn "" + exitWith . exitCodeForStatus $ reportStatus status + +{- | Check the testnet in the given output directory. Each node gets the +given number of seconds to answer, and all nodes are asked at the same +time. 'Left' means the check itself failed: the manifest, or the shelley +genesis it points to, cannot be read. +-} +checkStatus :: FilePath -> Nat -> IO (Either String StatusReport) +checkStatus outputDir timeoutSeconds = do + let manifestFile = outputDir defaultManifestFile + hasManifest <- doesFileExist manifestFile + if not hasManifest + then Right <$> noTestnetReport outputDir + else runExceptT $ do + manifest@Manifest{manifestPaths = ManifestPaths{mpGenesisFiles = ManifestGenesisFiles{mgfShelley = shelleyGenesisFile}}} <- + readJsonFile manifestFile + genesis <- readJsonFile $ outputDir shelleyGenesisFile + liftIO $ checkNodes outputDir timeoutSeconds manifest genesis + where + readJsonFile :: (FromJSON a) => String -> ExceptT String IO a + readJsonFile file = + ExceptT $ + either + (\err -> Left $ "cannot read " <> file <> ": " <> ioeGetErrorString err) + (mapLeft (\e -> "cannot decode " <> file <> ": " <> e)) + <$> tryIOError (eitherDecodeFileStrict' file) + +checkNodes :: FilePath -> Nat -> Manifest -> ShelleyGenesis -> IO StatusReport +checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = ManifestNetwork{mnMagic = magic}}) genesis = do + checkedAt <- getCurrentTime + nodes <- + forConcurrently (manifestNodes manifest) $ + probeNode outputDir timeoutSeconds magic limit slotStart + pure + StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = checkedAt + , reportStatus = classifyNetwork (probeNodeState <$> nodes) + , reportBestTip = listToMaybe . List.sortOn tipAge . mapMaybe probeNodeTipInfo $ toList nodes + , reportNodes = toList nodes + , reportProblemExplanation = Nothing + } + where + -- The chain watchdog's threshold for a dead chain: one number for both. + limit = chainStallTimeoutFromHorizon (chainForecastHorizon genesis) + slotLength = SL.fromNominalDiffTimeMicro (sgSlotLength genesis) + slotStart (SlotNo slot) = addUTCTime (fromIntegral slot * slotLength) (sgSystemStart genesis) + +{- | Check the status of one node in the testnet. +-} +probeNode :: + -- | output directory + FilePath -> + -- | timeout, in seconds + Nat -> + -- | network magic + Word32 -> + -- | tip age limit + NominalDiffTime -> + -- | start time of a slot + (SlotNo -> UTCTime) -> + ManifestNode -> + IO NodeProbeResult +probeNode + outputDir + timeoutSeconds + magic + limit + slotStart + ManifestNode + { mnodeName = nodeName + , mnodeRole = role + , mnodePid = pid + , mnodeSocketPath = socketPath + } = do + answer <- askTip + pidAlive <- maybe (pure Nothing) isProcessAlive pid + pure + NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = role + , probePid = pid + , probePidIsAlive = pidAlive + , probeNodeState = calcNodeState limit pidAlive answer + , probeNodeTipInfo = case answer of + AnsweredTip tip -> Just tip + AnsweredNoBlocks -> Nothing + NoAnswer -> Nothing } - ) + where + askTip :: IO NodeAnswer + askTip = do + -- Any failure (no socket, refused, handshake error) counts as no answer. + result <- + fromRight Nothing + <$> ( try $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) :: + IO (Either SomeException (Maybe ChainTip)) + ) + answeredAt <- getCurrentTime + pure $ case result of + Just (ChainTip slot hash blockNo) -> + AnsweredTip + TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipHash = hash + , tipAge = answeredAt `diffUTCTime` slotStart slot + } + Just ChainTipAtGenesis -> AnsweredNoBlocks + Nothing -> NoAnswer + + connectInfo :: LocalNodeConnectInfo + connectInfo = + LocalNodeConnectInfo + { localNodeSocketPath = mapFile (outputDir ) socketPath + , localNodeNetworkId = Testnet (NetworkMagic $ fromIntegral magic) + , localConsensusModeParams = CardanoModeParams testnetEpochSlots + } + +-- | The rule for the whole network. +classifyNetwork :: NonEmpty NodeState -> NetworkStatus +classifyNetwork states + | all (== NodeOk) states = NetworkRunning + | NodeOk `elem` states = NetworkDegraded + | NodeStalled `elem` states = NetworkStalled + | otherwise = NetworkStopped + +{- | The rule for one node. The answer decides; the pid only tells "down" +(the process is gone) from "unreachable" (alive, or unknown). +-} +calcNodeState :: + -- | tip age limit + NominalDiffTime -> + -- | whether the node's process is alive, if known + Maybe Bool -> + NodeAnswer -> + NodeState +calcNodeState limit _pidAlive (AnsweredTip tip) + | tipAge tip <= limit = NodeOk + | otherwise = NodeStalled +calcNodeState _limit _pidAlive AnsweredNoBlocks = NodeStalled +calcNodeState _limit pidAlive NoAnswer + | pidAlive == Just False = NodeDown + | otherwise = NodeUnreachable + +-- | The 'StatusReport' when there is no manifest, with a hint about why. +noTestnetReport :: FilePath -> IO StatusReport +noTestnetReport outputDir = do + now <- getCurrentTime + dirExists <- doesDirectoryExist outputDir + hasGenesis <- doesFileExist $ outputDir defaultGenesisFilepath ShelleyEra + let explanation + | not dirExists = "The directory " <> outputDir <> " does not exist." + | hasGenesis = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + | otherwise = outputDir <> " is not a testnet output directory." + pure + StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = now + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just explanation + } -renderStatusReportAsTable :: StatusReport -> String -renderStatusReportAsTable +-- | Render a 'StatusReport' in a human readable way. +renderStatusReportHumanReadable :: StatusReport -> String +renderStatusReportHumanReadable StatusReport { reportStatus = status , reportBestTip = bestTip @@ -119,6 +316,7 @@ renderStatusReportAsTable nodes ] +-- | Render an array of strings as a table, where each string in the output list is a line. renderTable :: [[String]] -> [String] renderTable rows = ("+" ++ concat ["-" ++ replicate width '-' ++ "-+" | width <- widths]) diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index 7bd98e53c86..adf9c92e9ae 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -4,11 +4,13 @@ module Testnet.Status.Types ( CheckStatusOptions (..) , NetworkStatus(..) + , NodeAnswer(..) , NodeProbeResult(..) , NodeState(..) , OutputFormat(..) , StatusReport(..) , TipInfo(..) + , exitCodeForStatus ) where import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), ToJSON (..)) @@ -79,6 +81,13 @@ instance ToJSON NodeState where toJSON NodeUnreachable = "unreachable" toJSON NodeDown = "down" +-- | What a node said when it was asked for its chain tip. +data NodeAnswer + = AnsweredTip !TipInfo -- ^ it has blocks; this is the newest + | AnsweredNoBlocks -- ^ it answered, but it has no blocks yet + | NoAnswer -- ^ it refused, failed, or did not answer in time + deriving (Eq, Show) + -- | A node's chain tip. data TipInfo = TipInfo { tipSlot :: !SlotNo @@ -117,11 +126,12 @@ instance ToJSON NodeProbeResult where , "tip" .= probeNodeTipInfo n ] +-- | Output of status command data StatusReport = StatusReport { reportOutputDir :: !FilePath , reportCheckedAt :: !UTCTime , reportStatus :: !NetworkStatus - , reportBestTip :: !(Maybe TipInfo) -- ^ the most tip of all nodes + , reportBestTip :: !(Maybe TipInfo) -- ^ the freshest tip of all nodes , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' , reportProblemExplanation :: !(Maybe String) -- ^ why there is no testnet } deriving (Eq, Show) @@ -143,10 +153,10 @@ instance ToJSON StatusReport where exitCodeNumber ExitSuccess = 0 exitCodeNumber (ExitFailure n) = n - -- | Convert the network state to an exit code - exitCodeForStatus :: NetworkStatus -> ExitCode - exitCodeForStatus NetworkRunning = ExitSuccess - exitCodeForStatus NetworkDegraded = ExitSuccess - exitCodeForStatus NetworkStalled = ExitFailure 3 - exitCodeForStatus NetworkStopped = ExitFailure 4 - exitCodeForStatus NoTestnet = ExitFailure 5 +-- | Convert the network state to an exit code +exitCodeForStatus :: NetworkStatus -> ExitCode +exitCodeForStatus NetworkRunning = ExitSuccess +exitCodeForStatus NetworkDegraded = ExitSuccess +exitCodeForStatus NetworkStalled = ExitFailure 3 +exitCodeForStatus NetworkStopped = ExitFailure 4 +exitCodeForStatus NoTestnet = ExitFailure 5 diff --git a/cardano-testnet/src/Testnet/Types.hs b/cardano-testnet/src/Testnet/Types.hs index 5cb0cd703b6..7170c4509b4 100644 --- a/cardano-testnet/src/Testnet/Types.hs +++ b/cardano-testnet/src/Testnet/Types.hs @@ -26,6 +26,7 @@ module Testnet.Types , nodeGrpcServer , nodeConnectionInfo , testnetNodeConnectionInfo + , testnetEpochSlots , isTestnetNodeSpo , SpoNodeKeys(..) , Delegator(..) @@ -222,9 +223,14 @@ testnetNodeConnectionInfo testnetMagic node = LocalNodeConnectInfo { localNodeSocketPath= nodeSocketPath node , localNodeNetworkId=Testnet (NetworkMagic $ fromIntegral testnetMagic) - , localConsensusModeParams=CardanoModeParams $ EpochSlots 21600} + , localConsensusModeParams=CardanoModeParams testnetEpochSlots } +-- | The Byron epoch size used when connecting to a testnet node. Shared with +-- @cardano-testnet status@, so that both connect in the same way. +testnetEpochSlots :: EpochSlots +testnetEpochSlots = EpochSlots 21600 + data SpoNodeKeys = SpoNodeKeys { poolNodeKeysCold :: KeyPair StakePoolKey , poolNodeKeysVrf :: KeyPair VrfKey From 9d4fc9cac1033167fdab0cebe4031305d13dc86b Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 20 Oct 2026 19:08:44 +0000 Subject: [PATCH 06/15] Add JSON schema for the status output and check it in CI --- .github/workflows/check-testnet-manifest.yml | 3 + .github/workflows/haskell.yml | 11 +- Makefile | 8 +- cardano-testnet/README.md | 18 +++ cardano-testnet/cardano-testnet.cabal | 1 + cardano-testnet/schemas/status.schema.json | 124 ++++++++++++++++++ cardano-testnet/src/Testnet/Status/Types.hs | 3 +- .../Cardano/Testnet/Test/Golden/Status.hs | 86 ++++++++++++ .../cardano-testnet-golden.hs | 2 + .../files/golden/status.json | 57 ++++++++ .../files/golden/status_no_testnet.json | 9 ++ nix/haskell.nix | 2 + 12 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 cardano-testnet/schemas/status.schema.json create mode 100644 cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status.json create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json diff --git a/.github/workflows/check-testnet-manifest.yml b/.github/workflows/check-testnet-manifest.yml index fb6825c66d5..eb504d686e3 100644 --- a/.github/workflows/check-testnet-manifest.yml +++ b/.github/workflows/check-testnet-manifest.yml @@ -34,3 +34,6 @@ jobs: - name: Validate cardano-testnet manifest golden file against its schema run: make testnet-manifest-validate + + - name: Validate cardano-testnet status golden files against their schema + run: make testnet-status-validate diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index b0457ab4d4f..1abd462ba71 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -142,7 +142,7 @@ jobs: run: | cabal test cardano-testnet cardano-node cardano-node-chairman cardano-submit-api - - name: Validate produced manifest files against schema + - name: Validate produced manifest and status output against schemas if: matrix.validate-manifest run: | # This job has no nix, that is why we pin the version with pipx separately. @@ -159,6 +159,9 @@ jobs: if ! kill -0 "$testnet_pid" 2>/dev/null; then break; fi sleep 5 done + # Exit code 0: the chain is producing blocks. + status_rc=0 + "$(cabal list-bin cardano-testnet)" status --testnet-dir "$dir" --json > "$RUNNER_TEMP/status.json" || status_rc=$? # SIGINT, not SIGTERM: the harness tears its nodes down on SIGINT only # (see Testnet.Signal.interruptNodesOnSigINT). kill -INT "$testnet_pid" 2>/dev/null || true @@ -169,6 +172,12 @@ jobs: exit 1 fi check-jsonschema --schemafile cardano-testnet/schemas/manifest.schema.json "$dir/manifest.json" + cat "$RUNNER_TEMP/status.json" + if [ "$status_rc" != 0 ]; then + echo "ERROR: cardano-testnet status exited with $status_rc" + exit 1 + fi + check-jsonschema --schemafile cardano-testnet/schemas/status.schema.json "$RUNNER_TEMP/status.json" - name: Tar failed tests workspaces if: ${{ failure() }} diff --git a/Makefile b/Makefile index 03753373428..7a4fef2be4f 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,12 @@ testnet-manifest-validate: ## Validate manifest golden file (and optionally MANI cardano-testnet/test/cardano-testnet-golden/files/golden/manifest.json $(if $(MANIFEST),nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/manifest.schema.json $(MANIFEST),) +testnet-status-validate: ## Validate status golden files (and optionally STATUS=path) against their schema + nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/status.schema.json \ + cardano-testnet/test/cardano-testnet-golden/files/golden/status.json \ + cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json + $(if $(STATUS),nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/status.schema.json $(STATUS),) + ### ### Workbench: cluster shells ### @@ -84,6 +90,6 @@ cls: .PHONY: help lint hlint host-hlint haddock-hoogle stylish-haskell cabal-hashes cli node \ trace-documentation trace-schemas-regenerate trace-schemas-overrides-check \ trace-schemas-overrides-coverage trace-schemas-validate \ - testnet-manifest-validate \ + testnet-manifest-validate testnet-status-validate \ workbench-ci ci ci-report ci-targets \ clean full-clean cls diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index 28f03b73347..da6c384e2d6 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -443,6 +443,24 @@ readers never see a partial file. It is written once and never modified. A pre-existing manifest from an earlier run in the same directory is deleted before nodes start. +### Status output + +`cardano-testnet status --json` prints the health of a testnet as a single +JSON object: the state of the network, the exit code of the command, the +freshest tip, and the state and tip of every node. The exit code is 0 while +the chain is producing blocks, even when some nodes are not healthy. A script +that needs every node to be healthy can check the `status` field: + +```bash +[ "$(cardano-testnet status --testnet-dir testnet --json | jq -r '.status')" = running ] +``` + +**Stability promise.** Like the manifest, the output has a strict JSON Schema +(`cardano-testnet/schemas/status.schema.json`, draft 2020-12, unknown fields +are an error) that lives in the repository and is the contract. Any change to +the output shape — including new fields — bumps the schema version, so the +output always matches the schema of its own version exactly. + ## Supported versions cardano-testnet is designed to run the cardano-cli and cardano-node that ship diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 6a115657752..b225fa9f680 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -174,6 +174,7 @@ test-suite cardano-testnet-golden other-modules: Cardano.Testnet.Test.Golden.Config Cardano.Testnet.Test.Golden.Help Cardano.Testnet.Test.Golden.Manifest + Cardano.Testnet.Test.Golden.Status Cardano.Testnet.Test.Golden.Util type: exitcode-stdio-1.0 diff --git a/cardano-testnet/schemas/status.schema.json b/cardano-testnet/schemas/status.schema.json new file mode 100644 index 00000000000..08f3adaa8c1 --- /dev/null +++ b/cardano-testnet/schemas/status.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/IntersectMBO/cardano-node/cardano-testnet/schemas/status.schema.json", + "title": "cardano-testnet status", + "description": "Output of 'cardano-testnet status --json': the health of a testnet, found by asking every node in its manifest for its chain tip. The schema is strict: it describes schema version 1 exactly, unknown fields are an error, and any change to the output shape bumps schemaVersion.", + "type": "object", + "required": [ + "schemaVersion", + "checkedAt", + "status", + "exitCode", + "chainProducingBlocks", + "bestTip", + "nodes" + ], + "additionalProperties": false, + "properties": { + "schemaVersion": { + "type": "integer", + "const": 1, + "description": "Schema version. Any change to the output shape, including new fields or values, bumps it." + }, + "checkedAt": { + "type": "string", + "format": "date-time", + "description": "UTC time the check started (ISO-8601)." + }, + "status": { + "type": "string", + "enum": ["running", "degraded", "stalled", "stopped", "no-testnet"], + "description": "State of the whole network. \"running\": every node is ok. \"degraded\": some nodes are ok, but not all. \"stalled\": some nodes answer, but none is ok. \"stopped\": no node answers. \"no-testnet\": there is no manifest in the directory." + }, + "exitCode": { + "type": "integer", + "enum": [0, 3, 4, 5], + "description": "Exit code of the command: 0 for \"running\" and \"degraded\", 3 for \"stalled\", 4 for \"stopped\", 5 for \"no-testnet\"." + }, + "chainProducingBlocks": { + "type": "boolean", + "description": "True when the exit code is 0: the status is \"running\" or \"degraded\"." + }, + "bestTip": { + "description": "The freshest tip of all nodes, or null when no node answered with a tip.", + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/tip" } + ] + }, + "nodes": { + "type": "array", + "description": "One entry per node, in the order of the manifest. Empty for \"no-testnet\".", + "items": { + "type": "object", + "required": ["name", "role", "pid", "pidAlive", "state", "tip"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Node name, as in the manifest." + }, + "role": { + "type": "string", + "enum": ["spo", "relay"], + "description": "\"spo\" for stake pool operators, \"relay\" for relay nodes, as in the manifest." + }, + "pid": { + "description": "OS process ID from the manifest, or null when the manifest has none.", + "oneOf": [ + { "type": "null" }, + { "type": "integer" } + ] + }, + "pidAlive": { + "description": "Whether a process with this pid exists, or null when that is unknown: no pid, or Windows.", + "oneOf": [ + { "type": "null" }, + { "type": "boolean" } + ] + }, + "state": { + "type": "string", + "enum": ["ok", "stalled", "unreachable", "down"], + "description": "\"ok\": it answers, and its tip is fresh. \"stalled\": it answers, but its tip is too old, or it has no blocks yet. \"unreachable\": it does not answer, but its process may be alive. \"down\": it does not answer, and its process is gone." + }, + "tip": { + "description": "The node's tip, or null when it did not answer with one.", + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/tip" } + ] + } + } + } + } + }, + "$defs": { + "tip": { + "type": "object", + "required": ["slot", "blockNo", "hash", "ageSeconds"], + "additionalProperties": false, + "properties": { + "slot": { + "type": "integer", + "minimum": 0, + "description": "Slot number of the tip." + }, + "blockNo": { + "type": "integer", + "minimum": 0, + "description": "Block number of the tip." + }, + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "Block header hash of the tip, hex-encoded." + }, + "ageSeconds": { + "type": "number", + "description": "Seconds from the start of the tip's slot until the node answered." + } + } + } + } +} diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index adf9c92e9ae..ddb62ef70d4 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -45,7 +45,8 @@ instance Default OutputFormat where -- Output types -- | Version of the JSON output, pinned by --- @cardano-testnet/schemas/status.schema.json@. +-- @cardano-testnet/schemas/status.schema.json@. Any change to the output +-- shape, including new fields, bumps it. statusSchemaVersion :: Int statusSchemaVersion = 1 diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs new file mode 100644 index 00000000000..310dc8f2d03 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -0,0 +1,86 @@ +{-# LANGUAGE OverloadedStrings #-} + +{- HLINT ignore "Use camelCase" -} + +module Cardano.Testnet.Test.Golden.Status + ( golden_StatusJson + ) where + +import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), + deserialiseFromRawBytesHex) + +import Prelude + +import Data.Aeson.Encode.Pretty (encodePretty) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Lazy as LBS +import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text +import Data.Time.Calendar (fromGregorian) +import Data.Time.Clock (UTCTime (..)) +import System.FilePath (()) + +import Testnet.Manifest (ManifestNodeRole (..)) +import Testnet.Status.Types + +import Hedgehog (Property) +import Hedgehog.Extras.Test.Base (propertyOnce) +import Hedgehog.Extras.Test.Golden (diffVsGoldenFile) +import qualified Hedgehog.Extras.Test.Process as H + +-- | A fixed sample report with one node in each state, so that every node +-- field, with and without a value, is pinned. +sampleReport :: StatusReport +sampleReport = StatusReport + { reportOutputDir = "testnet" + , reportCheckedAt = UTCTime (fromGregorian 2026 1 1) 60 + , reportStatus = NetworkDegraded + , reportBestTip = Just freshTip + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled (Just oldTip) + , NodeProbeResult "node3" RoleRelay Nothing Nothing NodeUnreachable Nothing + , NodeProbeResult "node4" RoleRelay (Just 12348) (Just False) NodeDown Nothing + ] + , reportProblemExplanation = Nothing + } + where + freshTip = TipInfo (SlotNo 1234) (BlockNo 56) + (sampleHash "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a") 0.5 + oldTip = TipInfo (SlotNo 400) (BlockNo 20) + (sampleHash "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5") 83.4 + +-- | A fixed sample report for a directory without a testnet: it has no nodes +-- and no tip, so the empty and null forms of the output are pinned. +sampleNoTestnetReport :: StatusReport +sampleNoTestnetReport = StatusReport + { reportOutputDir = "testnet" + , reportCheckedAt = UTCTime (fromGregorian 2026 1 1) 60 + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just "The directory testnet does not exist." + } + +-- | Parse a fixture hash, failing loudly on a typo in the literal. +sampleHash :: ByteString -> Hash BlockHeader +sampleHash h = + either (\e -> error $ "sampleReport: bad fixture hash: " <> show e) id $ + deserialiseFromRawBytesHex h + +goldenFile :: FilePath -> FilePath +goldenFile name = "cardano-testnet/test/cardano-testnet-golden/files/golden" name + +-- | The golden files are also checked against +-- @cardano-testnet/schemas/status.schema.json@ (@make testnet-status-validate@), +-- so together they check that the output matches the schema. +-- +-- Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusJson/"'@ +golden_StatusJson :: Property +golden_StatusJson = propertyOnce $ do + base <- H.getProjectBase + diffVsGoldenFile (encode sampleReport) $ base goldenFile "status.json" + diffVsGoldenFile (encode sampleNoTestnetReport) $ base goldenFile "status_no_testnet.json" + where + encode = Text.unpack . Text.decodeUtf8 . LBS.toStrict . encodePretty diff --git a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs index 34f78d90659..901dd726c75 100644 --- a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs +++ b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs @@ -8,6 +8,7 @@ import qualified Cardano.Crypto.Init as Crypto import qualified Cardano.Testnet.Test.Golden.Config import qualified Cardano.Testnet.Test.Golden.Help import qualified Cardano.Testnet.Test.Golden.Manifest +import qualified Cardano.Testnet.Test.Golden.Status import Prelude @@ -30,6 +31,7 @@ tests = pure $ T.testGroup "Golden tests" , ignoreOnWindows "golden_HelpReadme" Cardano.Testnet.Test.Golden.Help.golden_HelpReadme , H.testPropertyNamed "golden_VersionCmd" (fromString "golden_VersionCmd") Cardano.Testnet.Test.Golden.Help.golden_VersionCmd , H.testPropertyNamed "golden_Manifest" (fromString "golden_Manifest") Cardano.Testnet.Test.Golden.Manifest.golden_Manifest + , H.testPropertyNamed "golden_StatusJson" (fromString "golden_StatusJson") Cardano.Testnet.Test.Golden.Status.golden_StatusJson ] ingredients :: [T.Ingredient] diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json new file mode 100644 index 00000000000..9276fcba56a --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json @@ -0,0 +1,57 @@ +{ + "bestTip": { + "ageSeconds": 0.5, + "blockNo": 56, + "hash": "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a", + "slot": 1234 + }, + "chainProducingBlocks": true, + "checkedAt": "2026-01-01T00:01:00Z", + "exitCode": 0, + "nodes": [ + { + "name": "node1", + "pid": 12345, + "pidAlive": true, + "role": "spo", + "state": "ok", + "tip": { + "ageSeconds": 0.5, + "blockNo": 56, + "hash": "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a", + "slot": 1234 + } + }, + { + "name": "node2", + "pid": 12346, + "pidAlive": true, + "role": "relay", + "state": "stalled", + "tip": { + "ageSeconds": 83.4, + "blockNo": 20, + "hash": "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5", + "slot": 400 + } + }, + { + "name": "node3", + "pid": null, + "pidAlive": null, + "role": "relay", + "state": "unreachable", + "tip": null + }, + { + "name": "node4", + "pid": 12348, + "pidAlive": false, + "role": "relay", + "state": "down", + "tip": null + } + ], + "schemaVersion": 1, + "status": "degraded" +} \ No newline at end of file diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json new file mode 100644 index 00000000000..e66c5ec10e9 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json @@ -0,0 +1,9 @@ +{ + "bestTip": null, + "chainProducingBlocks": false, + "checkedAt": "2026-01-01T00:01:00Z", + "exitCode": 5, + "nodes": [], + "schemaVersion": 1, + "status": "no-testnet" +} \ No newline at end of file diff --git a/nix/haskell.nix b/nix/haskell.nix index 470fcca7fd9..41b266281f8 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -285,6 +285,8 @@ let "configuration/defaults/byron-mainnet" "cardano-testnet/test/cardano-testnet-golden/files/golden/manifest.json" "cardano-testnet/test/cardano-testnet-golden/files/golden/node_default_config.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json" "cardano-testnet/test/cardano-testnet-test/files/golden/tx.failed.response.json.golden" "cardano-testnet/test/cardano-testnet-test/files/input/sample-constitution.txt" "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json" From e80d0af724a3c61804661f2e61f436b35755af77 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 20 Oct 2026 19:15:58 +0000 Subject: [PATCH 07/15] Add tests for the status command --- cardano-testnet/cardano-testnet.cabal | 2 + cardano-testnet/src/Testnet/Status/Run.hs | 4 + .../Cardano/Testnet/Test/Golden/Status.hs | 12 +++ .../cardano-testnet-golden.hs | 1 + .../files/golden/status.txt | 12 +++ .../files/golden/status_no_testnet.txt | 2 + .../Cardano/Testnet/Test/Status.hs | 84 +++++++++++++++++++ .../Cardano/Testnet/Test/StatusRules.hs | 67 +++++++++++++++ .../cardano-testnet-test.hs | 13 +++ nix/haskell.nix | 2 + 10 files changed, 199 insertions(+) create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt create mode 100644 cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs create mode 100644 cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index b225fa9f680..ec160e57261 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -259,6 +259,8 @@ test-suite cardano-testnet-test Cardano.Testnet.Test.Node.Shutdown Cardano.Testnet.Test.Parser Cardano.Testnet.Test.Manifest + Cardano.Testnet.Test.Status + Cardano.Testnet.Test.StatusRules Cardano.Testnet.Test.MainnetParams Cardano.Testnet.Test.SanityCheck Cardano.Testnet.Test.RunTestnet diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 59fd67ae174..632a1afe5c8 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -2,6 +2,10 @@ module Testnet.Status.Run ( runCheckStatusOptions, + checkStatus, + calcNodeState, + classifyNetwork, + renderStatusReportHumanReadable, ) where import Cardano.Api (BlockNo (..), CardanoEra (..), ChainTip (..), diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs index 310dc8f2d03..aca2cc201fe 100644 --- a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -4,6 +4,7 @@ module Cardano.Testnet.Test.Golden.Status ( golden_StatusJson + , golden_StatusTable ) where import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), @@ -21,6 +22,7 @@ import Data.Time.Clock (UTCTime (..)) import System.FilePath (()) import Testnet.Manifest (ManifestNodeRole (..)) +import Testnet.Status.Run (renderStatusReportHumanReadable) import Testnet.Status.Types import Hedgehog (Property) @@ -84,3 +86,13 @@ golden_StatusJson = propertyOnce $ do diffVsGoldenFile (encode sampleNoTestnetReport) $ base goldenFile "status_no_testnet.json" where encode = Text.unpack . Text.decodeUtf8 . LBS.toStrict . encodePretty + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusTable/"'@ +golden_StatusTable :: Property +golden_StatusTable = propertyOnce $ do + base <- H.getProjectBase + diffVsGoldenFile (renderStatusReportHumanReadable sampleReport) $ + base goldenFile "status.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleNoTestnetReport) $ + base goldenFile "status_no_testnet.txt" diff --git a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs index 901dd726c75..81fe990d6e7 100644 --- a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs +++ b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs @@ -32,6 +32,7 @@ tests = pure $ T.testGroup "Golden tests" , H.testPropertyNamed "golden_VersionCmd" (fromString "golden_VersionCmd") Cardano.Testnet.Test.Golden.Help.golden_VersionCmd , H.testPropertyNamed "golden_Manifest" (fromString "golden_Manifest") Cardano.Testnet.Test.Golden.Manifest.golden_Manifest , H.testPropertyNamed "golden_StatusJson" (fromString "golden_StatusJson") Cardano.Testnet.Test.Golden.Status.golden_StatusJson + , H.testPropertyNamed "golden_StatusTable" (fromString "golden_StatusTable") Cardano.Testnet.Test.Golden.Status.golden_StatusTable ] ingredients :: [T.Ingredient] diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt new file mode 100644 index 00000000000..c3de1c57080 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt @@ -0,0 +1,12 @@ +Network: DEGRADED - chain is producing blocks, but 3 of 4 nodes are not ok ++-------+-------+-------------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node1 | spo | ok | 12345 | 1234 | 56 | 0.5s | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node2 | relay | stalled | 12346 | 400 | 20 | 83.4s | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node3 | relay | unreachable | - | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node4 | relay | down | 12348 | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt new file mode 100644 index 00000000000..58e7f5b84ee --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (The directory testnet does not exist.) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs new file mode 100644 index 00000000000..d7c36b7c21e --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -0,0 +1,84 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.Testnet.Test.Status + ( hprop_status + , hprop_status_no_testnet + ) where + +import Cardano.Api (CardanoEra (..)) + +import Cardano.Testnet (TestnetNode (..), TestnetRuntime (..), createAndRunTestnet, + defaultManifestFile, isTestnetNodeSpo, mkConf) + +import Prelude + +import Control.Monad (void) +import Data.Default.Class (def) +import Data.Either (isLeft) +import Data.Foldable (toList) +import Data.List (find) +import System.FilePath (()) +import qualified System.Process as Process + +import Testnet.Defaults (defaultGenesisFilepath) +import Testnet.Property.Util (integrationRetryWorkspace) +import Testnet.Signal (hardKillProcess) +import Testnet.Status.Run (checkStatus) +import Testnet.Status.Types + +import Hedgehog (Property, (===)) +import qualified Hedgehog as H +import qualified Hedgehog.Extras as H + +-- | Integration test: a fresh testnet is RUNNING. After one relay is +-- killed, it is DEGRADED, and the killed relay is down. +-- +-- Execute with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status reports network health/"'@ +hprop_status :: Property +hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaultWatchdog_ $ do + conf <- mkConf tmpDir + TestnetRuntime{testnetNodes} <- createAndRunTestnet def def conf + + running <- H.evalEither =<< H.evalIO (checkStatus tmpDir 5) + reportStatus running === NetworkRunning + map probeNodeState (reportNodes running) === map (const NodeOk) (toList testnetNodes) + + relay <- H.evalMaybe $ find (not . isTestnetNodeSpo) testnetNodes + H.evalIO $ do + hardKillProcess (nodeProcessHandle relay) + -- Wait for the process, so that its pid is gone and status can tell "down". + void . Process.waitForProcess $ nodeProcessHandle relay + + degraded <- H.evalEither =<< H.evalIO (checkStatus tmpDir 5) + reportStatus degraded === NetworkDegraded + killed <- H.evalMaybe $ find ((== nodeName relay) . probeNodeName) (reportNodes degraded) + probeNodeState killed === NodeDown + +-- | 'checkStatus' without a running testnet: a directory without a manifest +-- is NO-TESTNET, with a hint about why, and a broken manifest is an error. +-- +-- Execute with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status without a testnet/"'@ +hprop_status_no_testnet :: Property +hprop_status_no_testnet = H.propertyOnce . H.workspace "status-no-testnet" $ \dir -> do + let missingDir = dir "missing" + missing <- H.evalEither =<< H.evalIO (checkStatus missingDir 1) + reportStatus missing === NoTestnet + reportProblemExplanation missing === Just ("The directory " <> missingDir <> " does not exist.") + + notTestnet <- H.evalEither =<< H.evalIO (checkStatus dir 1) + reportStatus notTestnet === NoTestnet + reportProblemExplanation notTestnet === Just (dir <> " is not a testnet output directory.") + + -- The genesis files are written before the manifest, while the testnet starts. + H.writeFile (dir defaultGenesisFilepath ShelleyEra) "{}" + starting <- H.evalEither =<< H.evalIO (checkStatus dir 1) + reportStatus starting === NoTestnet + reportProblemExplanation starting + === Just "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + + H.writeFile (dir defaultManifestFile) "{" + broken <- H.evalIO $ checkStatus dir 1 + H.assertWith broken isLeft diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs new file mode 100644 index 00000000000..aa86b34dc8c --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs @@ -0,0 +1,67 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.Testnet.Test.StatusRules + ( unit_node_state + , unit_classify_network + , unit_exit_codes + ) where + +import Cardano.Api (BlockNo (..), SlotNo (..), deserialiseFromRawBytesHex) + +import Prelude + +import Data.List.NonEmpty (NonEmpty (..)) +import System.Exit (ExitCode (..)) + +import Testnet.Status.Run (calcNodeState, classifyNetwork) +import Testnet.Status.Types + +import Hedgehog ((===)) +import qualified Hedgehog as H +import qualified Hedgehog.Extras as H + +-- | A tip of the given age, in seconds. +tipOfAge :: Double -> TipInfo +tipOfAge age = TipInfo (SlotNo 100) (BlockNo 10) sampleHash (realToFrac age) + where + sampleHash = + either (\e -> error $ "tipOfAge: bad fixture hash: " <> show e) id $ + deserialiseFromRawBytesHex "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a" + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Node state/"'@ +unit_node_state :: H.Property +unit_node_state = H.propertyOnce $ do + let limit = 60 + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 1)) === NodeOk + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 60)) === NodeOk + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 61)) === NodeStalled + -- A node that answers but has no blocks yet is stalled. + calcNodeState limit (Just True) AnsweredNoBlocks === NodeStalled + -- The answer decides: a node that answers is never down, whatever its pid says. + calcNodeState limit (Just False) (AnsweredTip (tipOfAge 1)) === NodeOk + -- No answer: the pid tells down from unreachable. + calcNodeState limit (Just False) NoAnswer === NodeDown + calcNodeState limit (Just True) NoAnswer === NodeUnreachable + calcNodeState limit Nothing NoAnswer === NodeUnreachable + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Network status/"'@ +unit_classify_network :: H.Property +unit_classify_network = H.propertyOnce $ do + classifyNetwork (NodeOk :| [NodeOk, NodeOk]) === NetworkRunning + classifyNetwork (NodeOk :| [NodeDown, NodeUnreachable]) === NetworkDegraded + classifyNetwork (NodeOk :| [NodeStalled]) === NetworkDegraded + classifyNetwork (NodeStalled :| [NodeDown, NodeUnreachable]) === NetworkStalled + classifyNetwork (NodeDown :| [NodeUnreachable]) === NetworkStopped + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status exit codes/"'@ +unit_exit_codes :: H.Property +unit_exit_codes = H.propertyOnce $ do + -- 0 means the chain is producing blocks, even when some nodes are not ok. + exitCodeForStatus NetworkRunning === ExitSuccess + exitCodeForStatus NetworkDegraded === ExitSuccess + exitCodeForStatus NetworkStalled === ExitFailure 3 + exitCodeForStatus NetworkStopped === ExitFailure 4 + exitCodeForStatus NoTestnet === ExitFailure 5 diff --git a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs index 955728500db..5e857898adf 100644 --- a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs +++ b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs @@ -48,6 +48,8 @@ import qualified Cardano.Testnet.Test.Rpc.Transaction import qualified Cardano.Testnet.Test.RunTestnet import qualified Cardano.Testnet.Test.SanityCheck import qualified Cardano.Testnet.Test.SanityCheck as LedgerEvents +import qualified Cardano.Testnet.Test.Status +import qualified Cardano.Testnet.Test.StatusRules import qualified Cardano.Testnet.Test.SubmitApi.Transaction import qualified Cardano.Testnet.Test.UpdateTimeStamps @@ -154,6 +156,9 @@ tests = do , ignoreOnWindows "Stale manifest is removed by createTestnetEnv" Cardano.Testnet.Test.Manifest.hprop_manifest_stale , H.testPropertyNamed "Manifest Windows pipe path" (fromString "hprop_manifest_windows_pipe_path") Cardano.Testnet.Test.Manifest.hprop_manifest_windows_pipe_path + , ignoreOnWindows "Status reports network health" Cardano.Testnet.Test.Status.hprop_status + , H.testPropertyNamed "Status without a testnet" (fromString "hprop_status_no_testnet") + Cardano.Testnet.Test.Status.hprop_status_no_testnet ] , T.testGroup "SubmitApi" [ ignoreOnMacAndWindows "transaction" Cardano.Testnet.Test.SubmitApi.Transaction.hprop_transaction @@ -168,6 +173,14 @@ tests = do , ignoreOnWindows "RPC Transaction Submit" Cardano.Testnet.Test.Rpc.Transaction.hprop_rpc_transaction , ignoreOnWindows "RPC Eval Tx" Cardano.Testnet.Test.Rpc.Eval.hprop_rpc_eval_tx ] + , T.testGroup "Status rules" + [ H.testPropertyNamed "Node state" (fromString "unit_node_state") + Cardano.Testnet.Test.StatusRules.unit_node_state + , H.testPropertyNamed "Network status" (fromString "unit_classify_network") + Cardano.Testnet.Test.StatusRules.unit_classify_network + , H.testPropertyNamed "Status exit codes" (fromString "unit_exit_codes") + Cardano.Testnet.Test.StatusRules.unit_exit_codes + ] , T.testGroup "NodesWithOptions parser" [ H.testPropertyNamed "Roundtrip" (fromString "prop_parseNodeSpecs_roundtrip") Cardano.Testnet.Test.Parser.prop_parseNodeSpecs_roundtrip diff --git a/nix/haskell.nix b/nix/haskell.nix index 41b266281f8..62f1008bdb0 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -286,7 +286,9 @@ let "cardano-testnet/test/cardano-testnet-golden/files/golden/manifest.json" "cardano-testnet/test/cardano-testnet-golden/files/golden/node_default_config.json" "cardano-testnet/test/cardano-testnet-golden/files/golden/status.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt" "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt" "cardano-testnet/test/cardano-testnet-test/files/golden/tx.failed.response.json.golden" "cardano-testnet/test/cardano-testnet-test/files/input/sample-constitution.txt" "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json" From 7bf14541a68ebb1fb06f188f0869e6c0da4c4013 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 20 Oct 2026 21:37:01 +0200 Subject: [PATCH 08/15] Add changelog fragment --- ..._193638_cardano-testnet_pablo.lamela_status_command.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml diff --git a/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml new file mode 100644 index 00000000000..040848614b9 --- /dev/null +++ b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml @@ -0,0 +1,7 @@ +description: Added `status` command that checks the status of a testnet created with + `cardano-testnet`. The output can be either in human readable format or JSON (if + the `--json` flag is specified). +kind: +- feature +pr: 6741 +project: cardano-testnet From 5aa29b160391a7762d2e0bd08a2bf76f92b23fe9 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Thu, 22 Oct 2026 20:26:00 +0000 Subject: [PATCH 09/15] Pick the best tip by highest slot, not lowest age --- cardano-testnet/schemas/status.schema.json | 2 +- cardano-testnet/src/Testnet/Status/Run.hs | 11 ++++++++++- cardano-testnet/src/Testnet/Status/Types.hs | 2 +- .../Cardano/Testnet/Test/StatusRules.hs | 16 +++++++++++++++- .../cardano-testnet-test/cardano-testnet-test.hs | 2 ++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cardano-testnet/schemas/status.schema.json b/cardano-testnet/schemas/status.schema.json index 08f3adaa8c1..a26a76a308e 100644 --- a/cardano-testnet/schemas/status.schema.json +++ b/cardano-testnet/schemas/status.schema.json @@ -40,7 +40,7 @@ "description": "True when the exit code is 0: the status is \"running\" or \"degraded\"." }, "bestTip": { - "description": "The freshest tip of all nodes, or null when no node answered with a tip.", + "description": "The newest tip of all nodes: the one with the highest slot. Null when no node answered with a tip.", "oneOf": [ { "type": "null" }, { "$ref": "#/$defs/tip" } diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 632a1afe5c8..1d8226dfccd 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -5,6 +5,7 @@ module Testnet.Status.Run ( checkStatus, calcNodeState, classifyNetwork, + newestTip, renderStatusReportHumanReadable, ) where @@ -26,6 +27,7 @@ import Data.Either (fromRight) import Data.Either.Extra (mapLeft) import qualified Data.List as List import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) +import Data.Ord (Down (..)) import Data.Time (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime) import System.Directory (doesDirectoryExist, doesFileExist) import System.Exit (exitWith) @@ -104,7 +106,7 @@ checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = Manifes { reportOutputDir = outputDir , reportCheckedAt = checkedAt , reportStatus = classifyNetwork (probeNodeState <$> nodes) - , reportBestTip = listToMaybe . List.sortOn tipAge . mapMaybe probeNodeTipInfo $ toList nodes + , reportBestTip = newestTip . mapMaybe probeNodeTipInfo $ toList nodes , reportNodes = toList nodes , reportProblemExplanation = Nothing } @@ -211,6 +213,13 @@ calcNodeState _limit pidAlive NoAnswer | pidAlive == Just False = NodeDown | otherwise = NodeUnreachable +{- | The rule for the best tip: the newest one, which is the one with the +highest slot. The ages cannot tell: each age is measured when its node +answered, and the nodes answer at different times. +-} +newestTip :: [TipInfo] -> Maybe TipInfo +newestTip = listToMaybe . List.sortOn (Down . tipSlot) + -- | The 'StatusReport' when there is no manifest, with a hint about why. noTestnetReport :: FilePath -> IO StatusReport noTestnetReport outputDir = do diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index ddb62ef70d4..6472536b444 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -132,7 +132,7 @@ data StatusReport = StatusReport { reportOutputDir :: !FilePath , reportCheckedAt :: !UTCTime , reportStatus :: !NetworkStatus - , reportBestTip :: !(Maybe TipInfo) -- ^ the freshest tip of all nodes + , reportBestTip :: !(Maybe TipInfo) -- ^ the newest tip of all nodes: the one with the highest slot , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' , reportProblemExplanation :: !(Maybe String) -- ^ why there is no testnet } deriving (Eq, Show) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs index aa86b34dc8c..fc0ede1e78d 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs @@ -4,6 +4,7 @@ module Cardano.Testnet.Test.StatusRules ( unit_node_state , unit_classify_network , unit_exit_codes + , unit_best_tip ) where import Cardano.Api (BlockNo (..), SlotNo (..), deserialiseFromRawBytesHex) @@ -13,7 +14,7 @@ import Prelude import Data.List.NonEmpty (NonEmpty (..)) import System.Exit (ExitCode (..)) -import Testnet.Status.Run (calcNodeState, classifyNetwork) +import Testnet.Status.Run (calcNodeState, classifyNetwork, newestTip) import Testnet.Status.Types import Hedgehog ((===)) @@ -65,3 +66,16 @@ unit_exit_codes = H.propertyOnce $ do exitCodeForStatus NetworkStalled === ExitFailure 3 exitCodeForStatus NetworkStopped === ExitFailure 4 exitCodeForStatus NoTestnet === ExitFailure 5 + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Best tip/"'@ +unit_best_tip :: H.Property +unit_best_tip = H.propertyOnce $ do + -- With 0.1 s slots, node A answers at 10.2 s with slot 100, so its tip is + -- 0.2 s old. Node B answers later, at 12 s, with slot 110, so its tip is + -- 1 s old. B's tip is the newest, even though its age is bigger. + let tipA = (tipOfAge 0.2) { tipSlot = SlotNo 100 } + tipB = (tipOfAge 1) { tipSlot = SlotNo 110 } + newestTip [tipA, tipB] === Just tipB + newestTip [tipB, tipA] === Just tipB + newestTip [] === Nothing diff --git a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs index 5e857898adf..fbce4cf9d3b 100644 --- a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs +++ b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs @@ -180,6 +180,8 @@ tests = do Cardano.Testnet.Test.StatusRules.unit_classify_network , H.testPropertyNamed "Status exit codes" (fromString "unit_exit_codes") Cardano.Testnet.Test.StatusRules.unit_exit_codes + , H.testPropertyNamed "Best tip" (fromString "unit_best_tip") + Cardano.Testnet.Test.StatusRules.unit_best_tip ] , T.testGroup "NodesWithOptions parser" [ H.testPropertyNamed "Roundtrip" (fromString "prop_parseNodeSpecs_roundtrip") From 16020dab17baf3b7b15aff15773f125d64dba3dd Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Mon, 26 Oct 2026 22:49:02 +0000 Subject: [PATCH 10/15] Address small review comments on the status command --- .github/workflows/haskell.yml | 2 +- cardano-testnet/README.md | 2 +- cardano-testnet/src/Parsers/Run.hs | 2 +- cardano-testnet/src/Testnet/Signal.hs | 12 +++++++----- cardano-testnet/src/Testnet/Start/Types.hs | 6 ++++++ cardano-testnet/src/Testnet/Status/Parsers.hs | 4 +++- cardano-testnet/src/Testnet/Status/Run.hs | 16 +++++++++++----- cardano-testnet/src/Testnet/Status/Types.hs | 8 +++++--- .../Cardano/Testnet/Test/StatusRules.hs | 2 +- 9 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index 1abd462ba71..f4fef6cd1c5 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -162,6 +162,7 @@ jobs: # Exit code 0: the chain is producing blocks. status_rc=0 "$(cabal list-bin cardano-testnet)" status --testnet-dir "$dir" --json > "$RUNNER_TEMP/status.json" || status_rc=$? + cat "$RUNNER_TEMP/status.json" # SIGINT, not SIGTERM: the harness tears its nodes down on SIGINT only # (see Testnet.Signal.interruptNodesOnSigINT). kill -INT "$testnet_pid" 2>/dev/null || true @@ -172,7 +173,6 @@ jobs: exit 1 fi check-jsonschema --schemafile cardano-testnet/schemas/manifest.schema.json "$dir/manifest.json" - cat "$RUNNER_TEMP/status.json" if [ "$status_rc" != 0 ]; then echo "ERROR: cardano-testnet status exited with $status_rc" exit 1 diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index da6c384e2d6..46997505066 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -458,7 +458,7 @@ that needs every node to be healthy can check the `status` field: **Stability promise.** Like the manifest, the output has a strict JSON Schema (`cardano-testnet/schemas/status.schema.json`, draft 2020-12, unknown fields are an error) that lives in the repository and is the contract. Any change to -the output shape — including new fields — bumps the schema version, so the +the output shape, including new fields, bumps the schema version, so the output always matches the schema of its own version exactly. ## Supported versions diff --git a/cardano-testnet/src/Parsers/Run.hs b/cardano-testnet/src/Parsers/Run.hs index acc66b0b80b..0d36c211be9 100644 --- a/cardano-testnet/src/Parsers/Run.hs +++ b/cardano-testnet/src/Parsers/Run.hs @@ -102,7 +102,7 @@ runCardanoOptions = \case -- Create the sandbox, then run cardano-testnet. -- It is not necessary to update timestamps here, because -- the genesis files will be created with up-to-date stamps already. - let dirName = fromMaybe "testnet" noEnvOutputDir + let dirName = fromMaybe defaultTestnetDir noEnvOutputDir conf <- mkConfigAbs dirName runSimpleApp . runResourceT $ do logInfo $ "Creating environment: " <> display (tempAbsPath conf) diff --git a/cardano-testnet/src/Testnet/Signal.hs b/cardano-testnet/src/Testnet/Signal.hs index a3fc06b86af..b992a85921e 100644 --- a/cardano-testnet/src/Testnet/Signal.hs +++ b/cardano-testnet/src/Testnet/Signal.hs @@ -58,11 +58,13 @@ hardKillProcess = terminateProcess -- exists, and so does an unrelated process that reuses the pid. isProcessAlive :: Pid -> IO (Maybe Bool) #ifdef UNIX -isProcessAlive pid = do - result <- tryIOError $ signalProcess nullSignal pid - pure . Just $ case result of - Left e | isDoesNotExistError e -> False -- no such process - _ -> True -- signalled, or not allowed to: either way it exists +isProcessAlive pid + | pid <= 0 = pure (Just False) -- 0 and negative pids name process groups, not one process + | otherwise = do + result <- tryIOError $ signalProcess nullSignal pid + pure . Just $ case result of + Left e | isDoesNotExistError e -> False -- no such process + _ -> True -- signalled, or not allowed to: either way it exists #else isProcessAlive _ = pure Nothing #endif diff --git a/cardano-testnet/src/Testnet/Start/Types.hs b/cardano-testnet/src/Testnet/Start/Types.hs index adfa075aa33..66667e4faed 100644 --- a/cardano-testnet/src/Testnet/Start/Types.hs +++ b/cardano-testnet/src/Testnet/Start/Types.hs @@ -47,6 +47,7 @@ module Testnet.Start.Types , NodeConfigurationYaml , mkConf , mkConfigAbs + , defaultTestnetDir ) where import Cardano.Api hiding (cardanoEra) @@ -361,6 +362,11 @@ mkConfig tempAbsPath' = , updateTimestamps = DontUpdateTimestamps } +-- | The directory of the testnet when none is given: the @cardano@ command +-- creates it, and the @status@ command looks into it. +defaultTestnetDir :: FilePath +defaultTestnetDir = "./testnet" + -- | Create a 'Conf' from an absolute path, with updating time stamps disabled. mkConfigAbs :: FilePath -> IO Conf mkConfigAbs userOutputDir = do diff --git a/cardano-testnet/src/Testnet/Status/Parsers.hs b/cardano-testnet/src/Testnet/Status/Parsers.hs index fafa8e30af0..589195fc5b3 100644 --- a/cardano-testnet/src/Testnet/Status/Parsers.hs +++ b/cardano-testnet/src/Testnet/Status/Parsers.hs @@ -12,6 +12,7 @@ import Data.Default.Class (def) import Options.Applicative (CommandFields, Mod, Parser) import qualified Options.Applicative as OA +import Testnet.Start.Types (defaultTestnetDir) import Testnet.Status.Types (CheckStatusOptions (..), OutputFormat (..)) cmdCheckStatus :: Mod CommandFields CheckStatusOptions @@ -27,7 +28,8 @@ pOutputDir :: Parser (Maybe FilePath) pOutputDir = OA.optional (OA.strOption ( OA.long "testnet-dir" <> OA.metavar "FILEPATH" - <> OA.help "Path to the environment folder of the testnet whose status to check, by default \"./testnet\"." + <> OA.help ("Path to the environment folder of the testnet whose status to check, by default " + <> show defaultTestnetDir <> ".") )) diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 1d8226dfccd..dc3e0915e79 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -25,9 +25,10 @@ import Data.Aeson.Encode.Pretty (encodePretty) import qualified Data.ByteString.Lazy as LBS import Data.Either (fromRight) import Data.Either.Extra (mapLeft) -import qualified Data.List as List -import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) -import Data.Ord (Down (..)) +import Data.Foldable (maximumBy) +import Data.List.NonEmpty (nonEmpty) +import Data.Maybe (fromMaybe, mapMaybe) +import Data.Ord (comparing) import Data.Time (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime) import System.Directory (doesDirectoryExist, doesFileExist) import System.Exit (exitWith) @@ -41,6 +42,7 @@ import Testnet.ChainWatchdog (chainForecastHorizon, chainStallTimeoutF import Testnet.Manifest (Manifest (..), ManifestGenesisFiles (..), ManifestNetwork (..), ManifestNode (..), ManifestNodeRole (..), ManifestPaths (..)) import Testnet.Signal (isProcessAlive) +import Testnet.Start.Types (defaultTestnetDir) import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (..), NodeAnswer (..), NodeProbeResult (..), NodeState (..), OutputFormat (..), StatusReport (..), TipInfo (..), exitCodeForStatus) @@ -48,6 +50,10 @@ import Testnet.Types (sgSlotLength, sgSystemStart, testnetEpochSlots) import UnliftIO.Async (forConcurrently) +-- | Run the @status@ command: print the report (a table, or JSON) on stdout +-- and exit with 0 (running or degraded), 3 (stalled), 4 (stopped) or 5 (no +-- testnet). When the manifest or the genesis file cannot be read, print the +-- error on stderr and exit with 1. runCheckStatusOptions :: CheckStatusOptions -> IO () runCheckStatusOptions CheckStatusOptions @@ -55,7 +61,7 @@ runCheckStatusOptions , statusCheckTimeout = timeoutSeconds , outputFormat = format } = do - eStatus <- checkStatus (fromMaybe "./testnet" dir) timeoutSeconds + eStatus <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds status <- either ( \err -> do @@ -218,7 +224,7 @@ highest slot. The ages cannot tell: each age is measured when its node answered, and the nodes answer at different times. -} newestTip :: [TipInfo] -> Maybe TipInfo -newestTip = listToMaybe . List.sortOn (Down . tipSlot) +newestTip = fmap (maximumBy (comparing tipSlot)) . nonEmpty -- | The 'StatusReport' when there is no manifest, with a hint about why. noTestnetReport :: FilePath -> IO StatusReport diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index 6472536b444..948ec1d19a9 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -56,7 +56,9 @@ data NetworkStatus | NetworkDegraded -- ^ some nodes are 'NodeOk', but not all | NetworkStalled -- ^ some nodes answer, but none is 'NodeOk' | NetworkStopped -- ^ no node answers - | NoTestnet -- ^ there is no manifest file in the 'testnetDir' + | NoTestnet -- ^ there is no manifest file in the 'testnetDir': there is no + -- testnet, or it is still starting (the manifest is written + -- once the nodes are ready) deriving (Eq, Show) instance ToJSON NetworkStatus where @@ -93,8 +95,8 @@ data NodeAnswer data TipInfo = TipInfo { tipSlot :: !SlotNo , tipBlockNo :: !BlockNo - , tipHash :: Hash BlockHeader -- ^ hex - , tipAge :: !NominalDiffTime -- ^ time since the start of the tip's slot + , tipHash :: !(Hash BlockHeader) -- ^ hex + , tipAge :: !NominalDiffTime -- ^ time since the start of the tip's slot } deriving (Eq, Show) instance ToJSON TipInfo where diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs index fc0ede1e78d..af2674a3be3 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs @@ -27,7 +27,7 @@ tipOfAge age = TipInfo (SlotNo 100) (BlockNo 10) sampleHash (realToFrac age) where sampleHash = either (\e -> error $ "tipOfAge: bad fixture hash: " <> show e) id $ - deserialiseFromRawBytesHex "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a" + deserialiseFromRawBytesHex "3525888dc2c18eb5f5f34cb0edf911aa8b91ba485d195bb7d70dff7a7a32af91" -- | Execute me with: -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Node state/"'@ From d5811a53d2502dfbbd1c493671912564442fbf32 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 27 Oct 2026 01:53:40 +0000 Subject: [PATCH 11/15] Improve exception handling and report why nodes did not answer --- cardano-testnet/README.md | 3 +- cardano-testnet/schemas/status.schema.json | 9 ++- cardano-testnet/src/Testnet/Status/Run.hs | 68 ++++++++----------- cardano-testnet/src/Testnet/Status/Types.hs | 4 +- .../Cardano/Testnet/Test/Golden/Status.hs | 6 +- .../files/golden/status.json | 4 ++ .../files/golden/status.txt | 2 + .../Cardano/Testnet/Test/Status.hs | 16 +++-- .../Cardano/Testnet/Test/StatusRules.hs | 6 +- 9 files changed, 66 insertions(+), 52 deletions(-) diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index 46997505066..8a0ae4e6b3e 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -447,7 +447,8 @@ before nodes start. `cardano-testnet status --json` prints the health of a testnet as a single JSON object: the state of the network, the exit code of the command, the -freshest tip, and the state and tip of every node. The exit code is 0 while +freshest tip, and the state and tip of every node, with the error of a node +that did not answer. The exit code is 0 while the chain is producing blocks, even when some nodes are not healthy. A script that needs every node to be healthy can check the `status` field: diff --git a/cardano-testnet/schemas/status.schema.json b/cardano-testnet/schemas/status.schema.json index a26a76a308e..945e61a363c 100644 --- a/cardano-testnet/schemas/status.schema.json +++ b/cardano-testnet/schemas/status.schema.json @@ -51,7 +51,7 @@ "description": "One entry per node, in the order of the manifest. Empty for \"no-testnet\".", "items": { "type": "object", - "required": ["name", "role", "pid", "pidAlive", "state", "tip"], + "required": ["name", "role", "pid", "pidAlive", "state", "tip", "error"], "additionalProperties": false, "properties": { "name": { @@ -88,6 +88,13 @@ { "type": "null" }, { "$ref": "#/$defs/tip" } ] + }, + "error": { + "description": "Why the node did not answer (an error, or no answer in time), or null when it answered.", + "oneOf": [ + { "type": "null" }, + { "type": "string" } + ] } } } diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index dc3e0915e79..179a59df7f7 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -15,26 +15,22 @@ import Cardano.Api (BlockNo (..), CardanoEra (..), ChainTip (..), import qualified Cardano.Ledger.Shelley.API as SL import Cardano.Node.Testnet.Paths (defaultGenesisFilepath, defaultManifestFile) -import Cardano.Prelude (ExceptT (..), ExitCode (..), MonadIO (..), Nat, NonEmpty, - SomeException, Word32, runExceptT, toList, transpose, try) +import Cardano.Prelude (Nat, NonEmpty, Word32, toList, transpose) import Prelude import Data.Aeson (FromJSON, eitherDecodeFileStrict') import Data.Aeson.Encode.Pretty (encodePretty) import qualified Data.ByteString.Lazy as LBS -import Data.Either (fromRight) -import Data.Either.Extra (mapLeft) import Data.Foldable (maximumBy) import Data.List.NonEmpty (nonEmpty) import Data.Maybe (fromMaybe, mapMaybe) import Data.Ord (comparing) import Data.Time (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime) +import GHC.Stack (HasCallStack) import System.Directory (doesDirectoryExist, doesFileExist) import System.Exit (exitWith) import System.FilePath (()) -import qualified System.IO as IO -import System.IO.Error (ioeGetErrorString, tryIOError) import System.Timeout (timeout) import Text.Printf (printf) @@ -48,6 +44,7 @@ import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (. TipInfo (..), exitCodeForStatus) import Testnet.Types (sgSlotLength, sgSystemStart, testnetEpochSlots) +import RIO (displayException, throwString, tryAny) import UnliftIO.Async (forConcurrently) -- | Run the @status@ command: print the report (a table, or JSON) on stdout @@ -61,15 +58,7 @@ runCheckStatusOptions , statusCheckTimeout = timeoutSeconds , outputFormat = format } = do - eStatus <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds - status <- - either - ( \err -> do - IO.hPutStrLn IO.stderr $ "cardano-testnet status: " <> err - exitWith $ ExitFailure 1 - ) - pure - eStatus + status <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds case format of OutputFormatText -> putStr $ renderStatusReportHumanReadable status OutputFormatJson -> do LBS.putStr $ encodePretty status @@ -78,28 +67,25 @@ runCheckStatusOptions {- | Check the testnet in the given output directory. Each node gets the given number of seconds to answer, and all nodes are asked at the same -time. 'Left' means the check itself failed: the manifest, or the shelley -genesis it points to, cannot be read. +time. It throws when the check itself fails: when the manifest, or the +shelley genesis it points to, cannot be read. -} -checkStatus :: FilePath -> Nat -> IO (Either String StatusReport) +checkStatus :: HasCallStack => FilePath -> Nat -> IO StatusReport checkStatus outputDir timeoutSeconds = do let manifestFile = outputDir defaultManifestFile hasManifest <- doesFileExist manifestFile if not hasManifest - then Right <$> noTestnetReport outputDir - else runExceptT $ do + then noTestnetReport outputDir + else do manifest@Manifest{manifestPaths = ManifestPaths{mpGenesisFiles = ManifestGenesisFiles{mgfShelley = shelleyGenesisFile}}} <- readJsonFile manifestFile genesis <- readJsonFile $ outputDir shelleyGenesisFile - liftIO $ checkNodes outputDir timeoutSeconds manifest genesis + checkNodes outputDir timeoutSeconds manifest genesis where - readJsonFile :: (FromJSON a) => String -> ExceptT String IO a + readJsonFile :: (HasCallStack, FromJSON a) => String -> IO a readJsonFile file = - ExceptT $ - either - (\err -> Left $ "cannot read " <> file <> ": " <> ioeGetErrorString err) - (mapLeft (\e -> "cannot decode " <> file <> ": " <> e)) - <$> tryIOError (eitherDecodeFileStrict' file) + either (\e -> throwString $ "cannot decode " <> file <> ": " <> e) pure + =<< eitherDecodeFileStrict' file checkNodes :: FilePath -> Nat -> Manifest -> ShelleyGenesis -> IO StatusReport checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = ManifestNetwork{mnMagic = magic}}) genesis = do @@ -161,20 +147,19 @@ probeNode , probeNodeTipInfo = case answer of AnsweredTip tip -> Just tip AnsweredNoBlocks -> Nothing - NoAnswer -> Nothing + NoAnswer _ -> Nothing + , probeError = case answer of + NoAnswer reason -> Just reason + _ -> Nothing } where askTip :: IO NodeAnswer askTip = do -- Any failure (no socket, refused, handshake error) counts as no answer. - result <- - fromRight Nothing - <$> ( try $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) :: - IO (Either SomeException (Maybe ChainTip)) - ) + result <- tryAny $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) answeredAt <- getCurrentTime pure $ case result of - Just (ChainTip slot hash blockNo) -> + Right (Just (ChainTip slot hash blockNo)) -> AnsweredTip TipInfo { tipSlot = slot @@ -182,8 +167,9 @@ probeNode , tipHash = hash , tipAge = answeredAt `diffUTCTime` slotStart slot } - Just ChainTipAtGenesis -> AnsweredNoBlocks - Nothing -> NoAnswer + Right (Just ChainTipAtGenesis) -> AnsweredNoBlocks + Right Nothing -> NoAnswer $ "no answer within " <> show timeoutSeconds <> "s" + Left e -> NoAnswer $ displayException e connectInfo :: LocalNodeConnectInfo connectInfo = @@ -215,7 +201,7 @@ calcNodeState limit _pidAlive (AnsweredTip tip) | tipAge tip <= limit = NodeOk | otherwise = NodeStalled calcNodeState _limit _pidAlive AnsweredNoBlocks = NodeStalled -calcNodeState _limit pidAlive NoAnswer +calcNodeState _limit pidAlive (NoAnswer _) | pidAlive == Just False = NodeDown | otherwise = NodeUnreachable @@ -256,7 +242,7 @@ renderStatusReportHumanReadable , reportOutputDir = outputDir , reportProblemExplanation = problemExplanation } = - unlines $ ("Network: " <> summary status) : details + unlines $ ("Network: " <> summary status) : details <> errors where notOk = length $ filter ((/= NodeOk) . probeNodeState) nodes @@ -335,6 +321,12 @@ renderStatusReportHumanReadable nodes ] + errors :: [String] + errors = + [ nodeName <> ": " <> err + | NodeProbeResult{probeNodeName = nodeName, probeError = Just err} <- nodes + ] + -- | Render an array of strings as a table, where each string in the output list is a line. renderTable :: [[String]] -> [String] renderTable rows = diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index 948ec1d19a9..37bb147827d 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -88,7 +88,7 @@ instance ToJSON NodeState where data NodeAnswer = AnsweredTip !TipInfo -- ^ it has blocks; this is the newest | AnsweredNoBlocks -- ^ it answered, but it has no blocks yet - | NoAnswer -- ^ it refused, failed, or did not answer in time + | NoAnswer !String -- ^ it refused, failed, or did not answer in time: why deriving (Eq, Show) -- | A node's chain tip. @@ -116,6 +116,7 @@ data NodeProbeResult = NodeProbeResult , probePidIsAlive :: !(Maybe Bool) -- ^ 'Nothing' when unknown: no pid, or Windows , probeNodeState :: !NodeState , probeNodeTipInfo :: !(Maybe TipInfo) -- ^ 'Just' when the node answered with a tip + , probeError :: !(Maybe String) -- ^ 'Just' why, when the node did not answer } deriving (Eq, Show) instance ToJSON NodeProbeResult where @@ -127,6 +128,7 @@ instance ToJSON NodeProbeResult where , "pidAlive" .= probePidIsAlive n , "state" .= probeNodeState n , "tip" .= probeNodeTipInfo n + , "error" .= probeError n ] -- | Output of status command diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs index aca2cc201fe..191c30e8ad9 100644 --- a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -39,10 +39,12 @@ sampleReport = StatusReport , reportStatus = NetworkDegraded , reportBestTip = Just freshTip , reportNodes = - [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) - , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled (Just oldTip) + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled (Just oldTip) Nothing , NodeProbeResult "node3" RoleRelay Nothing Nothing NodeUnreachable Nothing + (Just "no answer within 5s") , NodeProbeResult "node4" RoleRelay (Just 12348) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") ] , reportProblemExplanation = Nothing } diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json index 9276fcba56a..67e1a41f679 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json @@ -10,6 +10,7 @@ "exitCode": 0, "nodes": [ { + "error": null, "name": "node1", "pid": 12345, "pidAlive": true, @@ -23,6 +24,7 @@ } }, { + "error": null, "name": "node2", "pid": 12346, "pidAlive": true, @@ -36,6 +38,7 @@ } }, { + "error": "no answer within 5s", "name": "node3", "pid": null, "pidAlive": null, @@ -44,6 +47,7 @@ "tip": null }, { + "error": "Network.Socket.connect: : does not exist (Connection refused)", "name": "node4", "pid": 12348, "pidAlive": false, diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt index c3de1c57080..66d9a286838 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt @@ -10,3 +10,5 @@ Network: DEGRADED - chain is producing blocks, but 3 of 4 nodes are not ok +-------+-------+-------------+-------+----------+-----------+---------+ | node4 | relay | down | 12348 | - | - | - | +-------+-------+-------------+-------+----------+-----------+---------+ +node3: no answer within 5s +node4: Network.Socket.connect: : does not exist (Connection refused) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs index d7c36b7c21e..02d82b64286 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -18,6 +18,7 @@ import Data.Default.Class (def) import Data.Either (isLeft) import Data.Foldable (toList) import Data.List (find) +import Data.Maybe (isJust) import System.FilePath (()) import qualified System.Process as Process @@ -31,6 +32,8 @@ import Hedgehog (Property, (===)) import qualified Hedgehog as H import qualified Hedgehog.Extras as H +import RIO (tryAny) + -- | Integration test: a fresh testnet is RUNNING. After one relay is -- killed, it is DEGRADED, and the killed relay is down. -- @@ -41,7 +44,7 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul conf <- mkConf tmpDir TestnetRuntime{testnetNodes} <- createAndRunTestnet def def conf - running <- H.evalEither =<< H.evalIO (checkStatus tmpDir 5) + running <- H.evalIO $ checkStatus tmpDir 5 reportStatus running === NetworkRunning map probeNodeState (reportNodes running) === map (const NodeOk) (toList testnetNodes) @@ -51,10 +54,11 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul -- Wait for the process, so that its pid is gone and status can tell "down". void . Process.waitForProcess $ nodeProcessHandle relay - degraded <- H.evalEither =<< H.evalIO (checkStatus tmpDir 5) + degraded <- H.evalIO $ checkStatus tmpDir 5 reportStatus degraded === NetworkDegraded killed <- H.evalMaybe $ find ((== nodeName relay) . probeNodeName) (reportNodes degraded) probeNodeState killed === NodeDown + H.assert . isJust $ probeError killed -- | 'checkStatus' without a running testnet: a directory without a manifest -- is NO-TESTNET, with a hint about why, and a broken manifest is an error. @@ -64,21 +68,21 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul hprop_status_no_testnet :: Property hprop_status_no_testnet = H.propertyOnce . H.workspace "status-no-testnet" $ \dir -> do let missingDir = dir "missing" - missing <- H.evalEither =<< H.evalIO (checkStatus missingDir 1) + missing <- H.evalIO $ checkStatus missingDir 1 reportStatus missing === NoTestnet reportProblemExplanation missing === Just ("The directory " <> missingDir <> " does not exist.") - notTestnet <- H.evalEither =<< H.evalIO (checkStatus dir 1) + notTestnet <- H.evalIO $ checkStatus dir 1 reportStatus notTestnet === NoTestnet reportProblemExplanation notTestnet === Just (dir <> " is not a testnet output directory.") -- The genesis files are written before the manifest, while the testnet starts. H.writeFile (dir defaultGenesisFilepath ShelleyEra) "{}" - starting <- H.evalEither =<< H.evalIO (checkStatus dir 1) + starting <- H.evalIO $ checkStatus dir 1 reportStatus starting === NoTestnet reportProblemExplanation starting === Just "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." H.writeFile (dir defaultManifestFile) "{" - broken <- H.evalIO $ checkStatus dir 1 + broken <- H.evalIO . tryAny $ checkStatus dir 1 H.assertWith broken isLeft diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs index af2674a3be3..11eaedaa491 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs @@ -42,9 +42,9 @@ unit_node_state = H.propertyOnce $ do -- The answer decides: a node that answers is never down, whatever its pid says. calcNodeState limit (Just False) (AnsweredTip (tipOfAge 1)) === NodeOk -- No answer: the pid tells down from unreachable. - calcNodeState limit (Just False) NoAnswer === NodeDown - calcNodeState limit (Just True) NoAnswer === NodeUnreachable - calcNodeState limit Nothing NoAnswer === NodeUnreachable + calcNodeState limit (Just False) (NoAnswer "refused") === NodeDown + calcNodeState limit (Just True) (NoAnswer "refused") === NodeUnreachable + calcNodeState limit Nothing (NoAnswer "refused") === NodeUnreachable -- | Execute me with: -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Network status/"'@ From 05c0e6e72dafc93d5caf9d28f01a3beb83e3dace Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 27 Oct 2026 01:53:40 +0000 Subject: [PATCH 12/15] Report why there is no testnet and document the exit codes --- ...no-testnet_pablo.lamela_status_command.yml | 5 ++++- cardano-testnet/README.md | 13 ++++++++++++- cardano-testnet/schemas/status.schema.json | 10 +++++++++- cardano-testnet/src/Testnet/Status/Run.hs | 19 ++++++++++++------- cardano-testnet/src/Testnet/Status/Types.hs | 17 ++++++++++++++++- .../Cardano/Testnet/Test/Golden/Status.hs | 8 +++++++- .../files/golden/status.json | 1 + .../files/golden/status_no_testnet.json | 1 + .../status_no_testnet_not_a_testnet_dir.txt | 2 ++ .../golden/status_no_testnet_not_ready.txt | 2 ++ .../Cardano/Testnet/Test/Status.hs | 7 +++---- nix/haskell.nix | 2 ++ 12 files changed, 71 insertions(+), 16 deletions(-) create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt diff --git a/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml index 040848614b9..a9997c87a24 100644 --- a/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml +++ b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml @@ -1,6 +1,9 @@ description: Added `status` command that checks the status of a testnet created with `cardano-testnet`. The output can be either in human readable format or JSON (if - the `--json` flag is specified). + the `--json` flag is specified). The library also exports the `Testnet.Status.Types`, + `Testnet.Status.Run` and `Testnet.Status.Parsers` modules, `isProcessAlive` from + `Testnet.Signal`, `testnetEpochSlots` from `Testnet.Types` and `defaultTestnetDir` + from `Testnet.Start.Types`. kind: - feature pr: 6741 diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index 8a0ae4e6b3e..963df286e21 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -446,7 +446,8 @@ before nodes start. ### Status output `cardano-testnet status --json` prints the health of a testnet as a single -JSON object: the state of the network, the exit code of the command, the +JSON object: the state of the network, the exit code of the command, why +there is no testnet (when there is none), the freshest tip, and the state and tip of every node, with the error of a node that did not answer. The exit code is 0 while the chain is producing blocks, even when some nodes are not healthy. A script @@ -456,6 +457,16 @@ that needs every node to be healthy can check the `status` field: [ "$(cardano-testnet status --testnet-dir testnet --json | jq -r '.status')" = running ] ``` +The exit code, for the table and for `--json`: + +| Exit code | Meaning | +|-----------|-----------------------------------------------------------| +| 0 | `running` or `degraded`: the chain is producing blocks | +| 1 | the command failed, for example on an unreadable manifest | +| 3 | `stalled`: some nodes answer, but none has a fresh tip | +| 4 | `stopped`: no node answers | +| 5 | `no-testnet`: there is no manifest in the directory | + **Stability promise.** Like the manifest, the output has a strict JSON Schema (`cardano-testnet/schemas/status.schema.json`, draft 2020-12, unknown fields are an error) that lives in the repository and is the contract. Any change to diff --git a/cardano-testnet/schemas/status.schema.json b/cardano-testnet/schemas/status.schema.json index 945e61a363c..a23266064d0 100644 --- a/cardano-testnet/schemas/status.schema.json +++ b/cardano-testnet/schemas/status.schema.json @@ -11,7 +11,8 @@ "exitCode", "chainProducingBlocks", "bestTip", - "nodes" + "nodes", + "noTestnetReason" ], "additionalProperties": false, "properties": { @@ -30,6 +31,13 @@ "enum": ["running", "degraded", "stalled", "stopped", "no-testnet"], "description": "State of the whole network. \"running\": every node is ok. \"degraded\": some nodes are ok, but not all. \"stalled\": some nodes answer, but none is ok. \"stopped\": no node answers. \"no-testnet\": there is no manifest in the directory." }, + "noTestnetReason": { + "description": "Why there is no testnet, when the status is \"no-testnet\". \"dir-missing\": the directory does not exist. \"not-ready\": it has genesis files but no manifest: the testnet is still starting, or it never became ready. \"not-a-testnet-dir\": it is not the output directory of a testnet. Null for any other status.", + "oneOf": [ + { "type": "null" }, + { "type": "string", "enum": ["dir-missing", "not-ready", "not-a-testnet-dir"] } + ] + }, "exitCode": { "type": "integer", "enum": [0, 3, 4, 5], diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 179a59df7f7..9f3c22aa3c2 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -39,9 +39,9 @@ import Testnet.Manifest (Manifest (..), ManifestGenesisFiles (..), Man ManifestNode (..), ManifestNodeRole (..), ManifestPaths (..)) import Testnet.Signal (isProcessAlive) import Testnet.Start.Types (defaultTestnetDir) -import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (..), NodeAnswer (..), - NodeProbeResult (..), NodeState (..), OutputFormat (..), StatusReport (..), - TipInfo (..), exitCodeForStatus) +import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (..), + NoTestnetReason (..), NodeAnswer (..), NodeProbeResult (..), NodeState (..), + OutputFormat (..), StatusReport (..), TipInfo (..), exitCodeForStatus) import Testnet.Types (sgSlotLength, sgSystemStart, testnetEpochSlots) import RIO (displayException, throwString, tryAny) @@ -219,9 +219,9 @@ noTestnetReport outputDir = do dirExists <- doesDirectoryExist outputDir hasGenesis <- doesFileExist $ outputDir defaultGenesisFilepath ShelleyEra let explanation - | not dirExists = "The directory " <> outputDir <> " does not exist." - | hasGenesis = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." - | otherwise = outputDir <> " is not a testnet output directory." + | not dirExists = DirMissing + | hasGenesis = NotReady + | otherwise = NotATestnetDir pure StatusReport { reportOutputDir = outputDir @@ -283,7 +283,7 @@ renderStatusReportHumanReadable details :: [String] details = case status of - NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> x <> ")") problemExplanation] + NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> explain x <> ")") problemExplanation] _ -> renderTable $ ["NODE", "ROLE", "STATE", "PID", "TIP SLOT", "TIP BLOCK", "TIP AGE"] @@ -321,6 +321,11 @@ renderStatusReportHumanReadable nodes ] + explain :: NoTestnetReason -> String + explain DirMissing = "The directory " <> outputDir <> " does not exist." + explain NotReady = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + explain NotATestnetDir = outputDir <> " is not a testnet output directory." + errors :: [String] errors = [ nodeName <> ": " <> err diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs index 37bb147827d..fc955fad342 100644 --- a/cardano-testnet/src/Testnet/Status/Types.hs +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -4,6 +4,7 @@ module Testnet.Status.Types ( CheckStatusOptions (..) , NetworkStatus(..) + , NoTestnetReason(..) , NodeAnswer(..) , NodeProbeResult(..) , NodeState(..) @@ -69,6 +70,19 @@ instance ToJSON NetworkStatus where toJSON NetworkStopped = "stopped" toJSON NoTestnet = "no-testnet" +-- | Why there is no manifest file in the 'testnetDir'. +data NoTestnetReason + = DirMissing -- ^ the directory does not exist + | NotReady -- ^ it has genesis files: the testnet is still starting, or it never became ready + | NotATestnetDir -- ^ it is not the output directory of a testnet + deriving (Eq, Show) + +instance ToJSON NoTestnetReason where + toJSON :: NoTestnetReason -> Value + toJSON DirMissing = "dir-missing" + toJSON NotReady = "not-ready" + toJSON NotATestnetDir = "not-a-testnet-dir" + -- | The state of one node. data NodeState = NodeOk -- ^ it answers, and its tip is fresh @@ -138,7 +152,7 @@ data StatusReport = StatusReport , reportStatus :: !NetworkStatus , reportBestTip :: !(Maybe TipInfo) -- ^ the newest tip of all nodes: the one with the highest slot , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' - , reportProblemExplanation :: !(Maybe String) -- ^ why there is no testnet + , reportProblemExplanation :: !(Maybe NoTestnetReason) -- ^ why there is no testnet } deriving (Eq, Show) instance ToJSON StatusReport where @@ -151,6 +165,7 @@ instance ToJSON StatusReport where , "chainProducingBlocks" .= (exitCodeForStatus (reportStatus r) == ExitSuccess) , "bestTip" .= reportBestTip r , "nodes" .= reportNodes r + , "noTestnetReason" .= reportProblemExplanation r ] where -- Extract code number from 'ExitCode' diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs index 191c30e8ad9..8f2369400e7 100644 --- a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -63,7 +63,7 @@ sampleNoTestnetReport = StatusReport , reportStatus = NoTestnet , reportBestTip = Nothing , reportNodes = [] - , reportProblemExplanation = Just "The directory testnet does not exist." + , reportProblemExplanation = Just DirMissing } -- | Parse a fixture hash, failing loudly on a typo in the literal. @@ -98,3 +98,9 @@ golden_StatusTable = propertyOnce $ do base goldenFile "status.txt" diffVsGoldenFile (renderStatusReportHumanReadable sampleNoTestnetReport) $ base goldenFile "status_no_testnet.txt" + diffVsGoldenFile + (renderStatusReportHumanReadable sampleNoTestnetReport{reportProblemExplanation = Just NotReady}) $ + base goldenFile "status_no_testnet_not_ready.txt" + diffVsGoldenFile + (renderStatusReportHumanReadable sampleNoTestnetReport{reportProblemExplanation = Just NotATestnetDir}) $ + base goldenFile "status_no_testnet_not_a_testnet_dir.txt" diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json index 67e1a41f679..7d8b35e03f9 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json @@ -8,6 +8,7 @@ "chainProducingBlocks": true, "checkedAt": "2026-01-01T00:01:00Z", "exitCode": 0, + "noTestnetReason": null, "nodes": [ { "error": null, diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json index e66c5ec10e9..94a7a2b4db9 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json @@ -3,6 +3,7 @@ "chainProducingBlocks": false, "checkedAt": "2026-01-01T00:01:00Z", "exitCode": 5, + "noTestnetReason": "dir-missing", "nodes": [], "schemaVersion": 1, "status": "no-testnet" diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt new file mode 100644 index 00000000000..dd6c9a75b33 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (testnet is not a testnet output directory.) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt new file mode 100644 index 00000000000..0b1d8446cb9 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (Genesis files exist, but no manifest: the testnet is still starting or it never became ready.) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs index 02d82b64286..a204b6ce61a 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -70,18 +70,17 @@ hprop_status_no_testnet = H.propertyOnce . H.workspace "status-no-testnet" $ \di let missingDir = dir "missing" missing <- H.evalIO $ checkStatus missingDir 1 reportStatus missing === NoTestnet - reportProblemExplanation missing === Just ("The directory " <> missingDir <> " does not exist.") + reportProblemExplanation missing === Just DirMissing notTestnet <- H.evalIO $ checkStatus dir 1 reportStatus notTestnet === NoTestnet - reportProblemExplanation notTestnet === Just (dir <> " is not a testnet output directory.") + reportProblemExplanation notTestnet === Just NotATestnetDir -- The genesis files are written before the manifest, while the testnet starts. H.writeFile (dir defaultGenesisFilepath ShelleyEra) "{}" starting <- H.evalIO $ checkStatus dir 1 reportStatus starting === NoTestnet - reportProblemExplanation starting - === Just "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + reportProblemExplanation starting === Just NotReady H.writeFile (dir defaultManifestFile) "{" broken <- H.evalIO . tryAny $ checkStatus dir 1 diff --git a/nix/haskell.nix b/nix/haskell.nix index 62f1008bdb0..1389848bca6 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -289,6 +289,8 @@ let "cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt" "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json" "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt" "cardano-testnet/test/cardano-testnet-test/files/golden/tx.failed.response.json.golden" "cardano-testnet/test/cardano-testnet-test/files/input/sample-constitution.txt" "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json" From 4aaf1e0d14a9e8db47d8da7bbe077325ffb3b709 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 27 Oct 2026 02:02:15 +0000 Subject: [PATCH 13/15] Test the status exit codes, the STOPPED state and the output of every state --- .../Cardano/Testnet/Test/Golden/Status.hs | 113 ++++++++++++++++-- .../cardano-testnet-golden.hs | 1 + .../files/golden/status_running.txt | 8 ++ .../files/golden/status_stalled.txt | 9 ++ .../files/golden/status_stalled_no_blocks.txt | 8 ++ .../files/golden/status_stopped.txt | 10 ++ .../Cardano/Testnet/Test/Status.hs | 17 ++- nix/haskell.nix | 4 + 8 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt create mode 100644 cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs index 8f2369400e7..044abd66bb2 100644 --- a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -5,28 +5,37 @@ module Cardano.Testnet.Test.Golden.Status ( golden_StatusJson , golden_StatusTable + , golden_StatusCommand ) where import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), deserialiseFromRawBytesHex) +import Cardano.Testnet.Test.Golden.Util (execDetailCardanoTestnet) + import Prelude +import Data.Aeson (Value (..), eitherDecodeFileStrict, eitherDecodeStrict) import Data.Aeson.Encode.Pretty (encodePretty) +import qualified Data.Aeson.KeyMap as KeyMap import Data.ByteString (ByteString) import qualified Data.ByteString.Lazy as LBS +import Data.List (isInfixOf) import qualified Data.Text as Text import qualified Data.Text.Encoding as Text import Data.Time.Calendar (fromGregorian) import Data.Time.Clock (UTCTime (..)) +import System.Exit (ExitCode (..)) import System.FilePath (()) import Testnet.Manifest (ManifestNodeRole (..)) import Testnet.Status.Run (renderStatusReportHumanReadable) import Testnet.Status.Types -import Hedgehog (Property) -import Hedgehog.Extras.Test.Base (propertyOnce) +import Hedgehog (Property, evalIO, (===)) +import Hedgehog.Extras.Test.Base (assertWith, leftFail, leftFailM, moduleWorkspace, + propertyOnce) +import qualified Hedgehog.Extras.Test.File as H import Hedgehog.Extras.Test.Golden (diffVsGoldenFile) import qualified Hedgehog.Extras.Test.Process as H @@ -48,11 +57,56 @@ sampleReport = StatusReport ] , reportProblemExplanation = Nothing } - where - freshTip = TipInfo (SlotNo 1234) (BlockNo 56) - (sampleHash "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a") 0.5 - oldTip = TipInfo (SlotNo 400) (BlockNo 20) - (sampleHash "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5") 83.4 + +-- | The tips of the sample reports: a fresh one and an old one. +freshTip, oldTip :: TipInfo +freshTip = TipInfo (SlotNo 1234) (BlockNo 56) + (sampleHash "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a") 0.5 +oldTip = TipInfo (SlotNo 400) (BlockNo 20) + (sampleHash "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5") 83.4 + +-- | Fixed sample reports for the other network states, for the text goldens. +sampleRunningReport :: StatusReport +sampleRunningReport = sampleReport + { reportStatus = NetworkRunning + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeOk (Just freshTip) Nothing + ] + } + +sampleStalledReport :: StatusReport +sampleStalledReport = sampleReport + { reportStatus = NetworkStalled + , reportBestTip = Just oldTip + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeStalled (Just oldTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") + ] + } + +sampleStalledNoBlocksReport :: StatusReport +sampleStalledNoBlocksReport = sampleReport + { reportStatus = NetworkStalled + , reportBestTip = Nothing + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeStalled Nothing Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled Nothing Nothing + ] + } + +sampleStoppedReport :: StatusReport +sampleStoppedReport = sampleReport + { reportStatus = NetworkStopped + , reportBestTip = Nothing + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") + , NodeProbeResult "node2" RoleRelay Nothing Nothing NodeUnreachable Nothing + (Just "no answer within 5s") + ] + } -- | A fixed sample report for a directory without a testnet: it has no nodes -- and no tip, so the empty and null forms of the output are pinned. @@ -104,3 +158,48 @@ golden_StatusTable = propertyOnce $ do diffVsGoldenFile (renderStatusReportHumanReadable sampleNoTestnetReport{reportProblemExplanation = Just NotATestnetDir}) $ base goldenFile "status_no_testnet_not_a_testnet_dir.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleRunningReport) $ + base goldenFile "status_running.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStalledReport) $ + base goldenFile "status_stalled.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStalledNoBlocksReport) $ + base goldenFile "status_stalled_no_blocks.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStoppedReport) $ + base goldenFile "status_stopped.txt" + +-- | The real command, without a testnet: the exit code, stdout and stderr for +-- a missing directory, as a table and as JSON, and for a broken manifest. The +-- JSON is the golden one, apart from the time of the check. +-- +-- Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusCommand/"'@ +golden_StatusCommand :: Property +golden_StatusCommand = propertyOnce . moduleWorkspace "status-command" $ \dir -> do + base <- H.getProjectBase + let missing = dir "missing" + + (code, out, err) <- execDetailCardanoTestnet ["status", "--testnet-dir", missing] + code === ExitFailure 5 + lines (filter (/= '\r') out) === + [ "Network: NO-TESTNET - no manifest.json in " <> missing + , "No testnet (The directory " <> missing <> " does not exist.)" + ] + err === "" + + (jsonCode, jsonOut, jsonErr) <- execDetailCardanoTestnet ["status", "--testnet-dir", missing, "--json"] + jsonCode === ExitFailure 5 + json <- leftFail (eitherDecodeStrict (Text.encodeUtf8 $ Text.pack jsonOut) :: Either String Value) + golden <- leftFailM . evalIO $ + (eitherDecodeFileStrict (base goldenFile "status_no_testnet.json") :: IO (Either String Value)) + withoutCheckedAt json === withoutCheckedAt golden + jsonErr === "" + + H.writeFile (dir "manifest.json") "{" + (brokenCode, brokenOut, brokenErr) <- execDetailCardanoTestnet ["status", "--testnet-dir", dir] + brokenCode === ExitFailure 1 + brokenOut === "" + assertWith brokenErr ("manifest.json" `isInfixOf`) + where + withoutCheckedAt :: Value -> Value + withoutCheckedAt (Object o) = Object (KeyMap.delete "checkedAt" o) + withoutCheckedAt v = v diff --git a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs index 81fe990d6e7..e52012aab56 100644 --- a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs +++ b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs @@ -33,6 +33,7 @@ tests = pure $ T.testGroup "Golden tests" , H.testPropertyNamed "golden_Manifest" (fromString "golden_Manifest") Cardano.Testnet.Test.Golden.Manifest.golden_Manifest , H.testPropertyNamed "golden_StatusJson" (fromString "golden_StatusJson") Cardano.Testnet.Test.Golden.Status.golden_StatusJson , H.testPropertyNamed "golden_StatusTable" (fromString "golden_StatusTable") Cardano.Testnet.Test.Golden.Status.golden_StatusTable + , H.testPropertyNamed "golden_StatusCommand" (fromString "golden_StatusCommand") Cardano.Testnet.Test.Golden.Status.golden_StatusCommand ] ingredients :: [T.Ingredient] diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt new file mode 100644 index 00000000000..c10d62d5365 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt @@ -0,0 +1,8 @@ +Network: RUNNING - chain is producing blocks (tip: slot 1234, block 56, 0.5s old) ++-------+-------+-------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------+-------+----------+-----------+---------+ +| node1 | spo | ok | 12345 | 1234 | 56 | 0.5s | ++-------+-------+-------+-------+----------+-----------+---------+ +| node2 | relay | ok | 12346 | 1234 | 56 | 0.5s | ++-------+-------+-------+-------+----------+-----------+---------+ diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt new file mode 100644 index 00000000000..b0fbd1af712 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt @@ -0,0 +1,9 @@ +Network: STALLED - chain is not producing blocks (tip is 83.4s old) ++-------+-------+---------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+---------+-------+----------+-----------+---------+ +| node1 | spo | stalled | 12345 | 400 | 20 | 83.4s | ++-------+-------+---------+-------+----------+-----------+---------+ +| node2 | relay | down | 12346 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ +node2: Network.Socket.connect: : does not exist (Connection refused) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt new file mode 100644 index 00000000000..8e2a4e16f11 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt @@ -0,0 +1,8 @@ +Network: STALLED - chain is not producing blocks (no node has a block yet) ++-------+-------+---------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+---------+-------+----------+-----------+---------+ +| node1 | spo | stalled | 12345 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ +| node2 | relay | stalled | 12346 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt new file mode 100644 index 00000000000..425d128dab0 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt @@ -0,0 +1,10 @@ +Network: STOPPED - no node answers ++-------+-------+-------------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node1 | spo | down | 12345 | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node2 | relay | unreachable | - | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +node1: Network.Socket.connect: : does not exist (Connection refused) +node2: no answer within 5s diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs index a204b6ce61a..9ba940a666f 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -13,7 +13,7 @@ import Cardano.Testnet (TestnetNode (..), TestnetRuntime (..), createA import Prelude -import Control.Monad (void) +import Control.Monad (forM_, void) import Data.Default.Class (def) import Data.Either (isLeft) import Data.Foldable (toList) @@ -34,8 +34,9 @@ import qualified Hedgehog.Extras as H import RIO (tryAny) --- | Integration test: a fresh testnet is RUNNING. After one relay is --- killed, it is DEGRADED, and the killed relay is down. +-- | Integration test: a fresh testnet is RUNNING, with a best tip. After one +-- relay is killed, it is DEGRADED, and the killed relay is down. After the +-- other nodes are killed too, it is STOPPED, and every node is down. -- -- Execute with: -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status reports network health/"'@ @@ -47,6 +48,7 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul running <- H.evalIO $ checkStatus tmpDir 5 reportStatus running === NetworkRunning map probeNodeState (reportNodes running) === map (const NodeOk) (toList testnetNodes) + H.assert . isJust $ reportBestTip running relay <- H.evalMaybe $ find (not . isTestnetNodeSpo) testnetNodes H.evalIO $ do @@ -60,6 +62,15 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul probeNodeState killed === NodeDown H.assert . isJust $ probeError killed + H.evalIO . forM_ (filter ((/= nodeName relay) . nodeName) $ toList testnetNodes) $ \node -> do + hardKillProcess (nodeProcessHandle node) + void . Process.waitForProcess $ nodeProcessHandle node + + stopped <- H.evalIO $ checkStatus tmpDir 5 + reportStatus stopped === NetworkStopped + map probeNodeState (reportNodes stopped) === map (const NodeDown) (toList testnetNodes) + reportBestTip stopped === Nothing + -- | 'checkStatus' without a running testnet: a directory without a manifest -- is NO-TESTNET, with a hint about why, and a broken manifest is an error. -- diff --git a/nix/haskell.nix b/nix/haskell.nix index 1389848bca6..b9b09cf536b 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -291,6 +291,10 @@ let "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt" "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt" "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt" "cardano-testnet/test/cardano-testnet-test/files/golden/tx.failed.response.json.golden" "cardano-testnet/test/cardano-testnet-test/files/input/sample-constitution.txt" "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json" From e37a89c482dedaac581a3a611456ca5f8b8c4ed1 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 27 Oct 2026 03:52:18 +0100 Subject: [PATCH 14/15] Improve consistency of formatting --- cardano-testnet/src/Testnet/Status/Run.hs | 419 +++++++++++----------- 1 file changed, 201 insertions(+), 218 deletions(-) diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 9f3c22aa3c2..697a09f401e 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -53,63 +53,59 @@ import UnliftIO.Async (forConcurrently) -- error on stderr and exit with 1. runCheckStatusOptions :: CheckStatusOptions -> IO () runCheckStatusOptions - CheckStatusOptions - { testnetDir = dir - , statusCheckTimeout = timeoutSeconds - , outputFormat = format - } = do - status <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds - case format of - OutputFormatText -> putStr $ renderStatusReportHumanReadable status - OutputFormatJson -> do LBS.putStr $ encodePretty status - putStrLn "" - exitWith . exitCodeForStatus $ reportStatus status + CheckStatusOptions + { testnetDir = dir + , statusCheckTimeout = timeoutSeconds + , outputFormat = format + } = do + status <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds + case format of + OutputFormatText -> putStr $ renderStatusReportHumanReadable status + OutputFormatJson -> do LBS.putStr $ encodePretty status + putStrLn "" + exitWith . exitCodeForStatus $ reportStatus status -{- | Check the testnet in the given output directory. Each node gets the -given number of seconds to answer, and all nodes are asked at the same -time. It throws when the check itself fails: when the manifest, or the -shelley genesis it points to, cannot be read. --} +-- | Check the testnet in the given output directory. Each node gets the +-- given number of seconds to answer, and all nodes are asked at the same +-- time. It throws when the check itself fails: when the manifest, or the +-- shelley genesis it points to, cannot be read. checkStatus :: HasCallStack => FilePath -> Nat -> IO StatusReport checkStatus outputDir timeoutSeconds = do - let manifestFile = outputDir defaultManifestFile - hasManifest <- doesFileExist manifestFile - if not hasManifest - then noTestnetReport outputDir - else do - manifest@Manifest{manifestPaths = ManifestPaths{mpGenesisFiles = ManifestGenesisFiles{mgfShelley = shelleyGenesisFile}}} <- - readJsonFile manifestFile - genesis <- readJsonFile $ outputDir shelleyGenesisFile - checkNodes outputDir timeoutSeconds manifest genesis - where - readJsonFile :: (HasCallStack, FromJSON a) => String -> IO a - readJsonFile file = - either (\e -> throwString $ "cannot decode " <> file <> ": " <> e) pure - =<< eitherDecodeFileStrict' file + let manifestFile = outputDir defaultManifestFile + hasManifest <- doesFileExist manifestFile + if not hasManifest + then noTestnetReport outputDir + else do + manifest@Manifest{manifestPaths = ManifestPaths{mpGenesisFiles = ManifestGenesisFiles{mgfShelley = shelleyGenesisFile}}} + <- readJsonFile manifestFile + genesis <- readJsonFile $ outputDir shelleyGenesisFile + checkNodes outputDir timeoutSeconds manifest genesis + where + readJsonFile :: (HasCallStack, FromJSON a) => String -> IO a + readJsonFile file = + either (\e -> throwString $ "cannot decode " <> file <> ": " <> e) pure + =<< eitherDecodeFileStrict' file checkNodes :: FilePath -> Nat -> Manifest -> ShelleyGenesis -> IO StatusReport checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = ManifestNetwork{mnMagic = magic}}) genesis = do - checkedAt <- getCurrentTime - nodes <- - forConcurrently (manifestNodes manifest) $ - probeNode outputDir timeoutSeconds magic limit slotStart - pure - StatusReport - { reportOutputDir = outputDir - , reportCheckedAt = checkedAt - , reportStatus = classifyNetwork (probeNodeState <$> nodes) - , reportBestTip = newestTip . mapMaybe probeNodeTipInfo $ toList nodes - , reportNodes = toList nodes - , reportProblemExplanation = Nothing - } + checkedAt <- getCurrentTime + nodes <- forConcurrently (manifestNodes manifest) $ + probeNode outputDir timeoutSeconds magic limit slotStart + pure StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = checkedAt + , reportStatus = classifyNetwork (probeNodeState <$> nodes) + , reportBestTip = newestTip . mapMaybe probeNodeTipInfo $ toList nodes + , reportNodes = toList nodes + , reportProblemExplanation = Nothing + } where -- The chain watchdog's threshold for a dead chain: one number for both. limit = chainStallTimeoutFromHorizon (chainForecastHorizon genesis) slotLength = SL.fromNominalDiffTimeMicro (sgSlotLength genesis) slotStart (SlotNo slot) = addUTCTime (fromIntegral slot * slotLength) (sgSystemStart genesis) -{- | Check the status of one node in the testnet. --} +-- | Check the status of one node in the testnet. probeNode :: -- | output directory FilePath -> @@ -123,73 +119,64 @@ probeNode :: (SlotNo -> UTCTime) -> ManifestNode -> IO NodeProbeResult -probeNode - outputDir - timeoutSeconds - magic - limit - slotStart - ManifestNode - { mnodeName = nodeName - , mnodeRole = role - , mnodePid = pid - , mnodeSocketPath = socketPath - } = do - answer <- askTip - pidAlive <- maybe (pure Nothing) isProcessAlive pid - pure - NodeProbeResult - { probeNodeName = nodeName - , probeNodeRole = role - , probePid = pid - , probePidIsAlive = pidAlive - , probeNodeState = calcNodeState limit pidAlive answer - , probeNodeTipInfo = case answer of - AnsweredTip tip -> Just tip - AnsweredNoBlocks -> Nothing - NoAnswer _ -> Nothing - , probeError = case answer of - NoAnswer reason -> Just reason - _ -> Nothing - } - where - askTip :: IO NodeAnswer - askTip = do - -- Any failure (no socket, refused, handshake error) counts as no answer. - result <- tryAny $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) - answeredAt <- getCurrentTime - pure $ case result of - Right (Just (ChainTip slot hash blockNo)) -> - AnsweredTip - TipInfo - { tipSlot = slot - , tipBlockNo = blockNo - , tipHash = hash - , tipAge = answeredAt `diffUTCTime` slotStart slot - } - Right (Just ChainTipAtGenesis) -> AnsweredNoBlocks - Right Nothing -> NoAnswer $ "no answer within " <> show timeoutSeconds <> "s" - Left e -> NoAnswer $ displayException e +probeNode outputDir timeoutSeconds magic limit slotStart + ManifestNode + { mnodeName = nodeName + , mnodeRole = role + , mnodePid = pid + , mnodeSocketPath = socketPath + } = do + answer <- askTip + pidAlive <- maybe (pure Nothing) isProcessAlive pid + pure NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = role + , probePid = pid + , probePidIsAlive = pidAlive + , probeNodeState = calcNodeState limit pidAlive answer + , probeNodeTipInfo = case answer of + AnsweredTip tip -> Just tip + AnsweredNoBlocks -> Nothing + NoAnswer _ -> Nothing + , probeError = case answer of + NoAnswer reason -> Just reason + _ -> Nothing + } + where + askTip :: IO NodeAnswer + askTip = do + -- Any failure (no socket, refused, handshake error) counts as no answer. + result <- tryAny $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) + answeredAt <- getCurrentTime + pure $ case result of + Right (Just (ChainTip slot hash blockNo)) -> + AnsweredTip TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipHash = hash + , tipAge = answeredAt `diffUTCTime` slotStart slot + } + Right (Just ChainTipAtGenesis) -> AnsweredNoBlocks + Right Nothing -> NoAnswer $ "no answer within " <> show timeoutSeconds <> "s" + Left e -> NoAnswer $ displayException e - connectInfo :: LocalNodeConnectInfo - connectInfo = - LocalNodeConnectInfo - { localNodeSocketPath = mapFile (outputDir ) socketPath - , localNodeNetworkId = Testnet (NetworkMagic $ fromIntegral magic) - , localConsensusModeParams = CardanoModeParams testnetEpochSlots - } + connectInfo :: LocalNodeConnectInfo + connectInfo = LocalNodeConnectInfo + { localNodeSocketPath = mapFile (outputDir ) socketPath + , localNodeNetworkId = Testnet (NetworkMagic $ fromIntegral magic) + , localConsensusModeParams = CardanoModeParams testnetEpochSlots + } -- | The rule for the whole network. classifyNetwork :: NonEmpty NodeState -> NetworkStatus classifyNetwork states - | all (== NodeOk) states = NetworkRunning - | NodeOk `elem` states = NetworkDegraded - | NodeStalled `elem` states = NetworkStalled - | otherwise = NetworkStopped + | all (== NodeOk) states = NetworkRunning + | NodeOk `elem` states = NetworkDegraded + | NodeStalled `elem` states = NetworkStalled + | otherwise = NetworkStopped -{- | The rule for one node. The answer decides; the pid only tells "down" -(the process is gone) from "unreachable" (alive, or unknown). --} +-- | The rule for one node. The answer decides; the pid only tells "down" +-- (the process is gone) from "unreachable" (alive, or unknown). calcNodeState :: -- | tip age limit NominalDiffTime -> @@ -205,137 +192,132 @@ calcNodeState _limit pidAlive (NoAnswer _) | pidAlive == Just False = NodeDown | otherwise = NodeUnreachable -{- | The rule for the best tip: the newest one, which is the one with the -highest slot. The ages cannot tell: each age is measured when its node -answered, and the nodes answer at different times. --} +-- | The rule for the best tip: the newest one, which is the one with the +-- highest slot. The ages cannot tell: each age is measured when its node +-- answered, and the nodes answer at different times. newestTip :: [TipInfo] -> Maybe TipInfo newestTip = fmap (maximumBy (comparing tipSlot)) . nonEmpty -- | The 'StatusReport' when there is no manifest, with a hint about why. noTestnetReport :: FilePath -> IO StatusReport noTestnetReport outputDir = do - now <- getCurrentTime - dirExists <- doesDirectoryExist outputDir - hasGenesis <- doesFileExist $ outputDir defaultGenesisFilepath ShelleyEra - let explanation - | not dirExists = DirMissing - | hasGenesis = NotReady - | otherwise = NotATestnetDir - pure - StatusReport - { reportOutputDir = outputDir - , reportCheckedAt = now - , reportStatus = NoTestnet - , reportBestTip = Nothing - , reportNodes = [] - , reportProblemExplanation = Just explanation - } + now <- getCurrentTime + dirExists <- doesDirectoryExist outputDir + hasGenesis <- doesFileExist $ outputDir defaultGenesisFilepath ShelleyEra + let explanation + | not dirExists = DirMissing + | hasGenesis = NotReady + | otherwise = NotATestnetDir + pure StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = now + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just explanation + } -- | Render a 'StatusReport' in a human readable way. renderStatusReportHumanReadable :: StatusReport -> String renderStatusReportHumanReadable - StatusReport - { reportStatus = status - , reportBestTip = bestTip - , reportNodes = nodes - , reportOutputDir = outputDir - , reportProblemExplanation = problemExplanation - } = - unlines $ ("Network: " <> summary status) : details <> errors - where - notOk = length $ filter ((/= NodeOk) . probeNodeState) nodes + StatusReport + { reportStatus = status + , reportBestTip = bestTip + , reportNodes = nodes + , reportOutputDir = outputDir + , reportProblemExplanation = problemExplanation + } = + unlines $ ("Network: " <> summary status) : details <> errors + where + notOk = length $ filter ((/= NodeOk) . probeNodeState) nodes - seconds :: NominalDiffTime -> String - seconds s = printf "%.1fs" (realToFrac s :: Double) + seconds :: NominalDiffTime -> String + seconds s = printf "%.1fs" (realToFrac s :: Double) - summary :: NetworkStatus -> String - summary NetworkRunning = - "RUNNING - chain is producing blocks" - <> foldMap - ( \TipInfo - { tipSlot = slotNo - , tipBlockNo = blockNo - , tipAge = age - } -> - " (tip: slot " - <> show (unSlotNo slotNo) - <> ", block " - <> show (unBlockNo blockNo) - <> ", " - <> seconds age - <> " old)" - ) - bestTip - summary NetworkDegraded = - "DEGRADED - chain is producing blocks, but " - <> show notOk - <> " of " - <> show (length nodes) - <> (if notOk == 1 then " nodes is" else " nodes are") - <> " not ok" - summary NetworkStalled = - "STALLED - chain is not producing blocks" <> case bestTip of - Just t -> " (tip is " <> seconds (tipAge t) <> " old)" - _ -> " (no node has a block yet)" - summary NetworkStopped = "STOPPED - no node answers" - summary NoTestnet = "NO-TESTNET - no " <> defaultManifestFile <> " in " <> outputDir + summary :: NetworkStatus -> String + summary NetworkRunning = + "RUNNING - chain is producing blocks" + <> foldMap + ( \TipInfo + { tipSlot = slotNo + , tipBlockNo = blockNo + , tipAge = age + } -> " (tip: slot " + <> show (unSlotNo slotNo) + <> ", block " + <> show (unBlockNo blockNo) + <> ", " + <> seconds age + <> " old)" + ) + bestTip + summary NetworkDegraded = + "DEGRADED - chain is producing blocks, but " + <> show notOk + <> " of " + <> show (length nodes) + <> (if notOk == 1 then " nodes is" else " nodes are") + <> " not ok" + summary NetworkStalled = + "STALLED - chain is not producing blocks" + <> case bestTip of + Just t -> " (tip is " <> seconds (tipAge t) <> " old)" + _ -> " (no node has a block yet)" + summary NetworkStopped = "STOPPED - no node answers" + summary NoTestnet = "NO-TESTNET - no " <> defaultManifestFile <> " in " <> outputDir - details :: [String] - details = case status of - NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> explain x <> ")") problemExplanation] - _ -> - renderTable $ - ["NODE", "ROLE", "STATE", "PID", "TIP SLOT", "TIP BLOCK", "TIP AGE"] - : [ [ nodeName - , case nodeRole of - RoleSpo -> "spo" - RoleRelay -> "relay" - , case nodeState of - NodeOk -> "ok" - NodeStalled -> "stalled" - NodeUnreachable -> "unreachable" - NodeDown -> "down" - , maybe "-" show mNodePid - ] - <> case mNodeTipInfo of - Nothing -> ["-", "-", "-"] - Just - ( TipInfo - { tipSlot = slot - , tipBlockNo = blockNo - , tipAge = age - } - ) -> - [ show $ unSlotNo slot - , show $ unBlockNo blockNo - , seconds age - ] - | NodeProbeResult - { probeNodeName = nodeName - , probeNodeRole = nodeRole - , probePid = mNodePid - , probeNodeState = nodeState - , probeNodeTipInfo = mNodeTipInfo - } <- - nodes - ] + details :: [String] + details = + case status of + NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> explain x <> ")") problemExplanation] + _ -> renderTable $ + ["NODE", "ROLE", "STATE", "PID", "TIP SLOT", "TIP BLOCK", "TIP AGE"] + : [ [ nodeName + , case nodeRole of + RoleSpo -> "spo" + RoleRelay -> "relay" + , case nodeState of + NodeOk -> "ok" + NodeStalled -> "stalled" + NodeUnreachable -> "unreachable" + NodeDown -> "down" + , maybe "-" show mNodePid + ] <> case mNodeTipInfo of + Nothing -> ["-", "-", "-"] + Just ( TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipAge = age + } + ) -> [ show $ unSlotNo slot + , show $ unBlockNo blockNo + , seconds age + ] + | NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = nodeRole + , probePid = mNodePid + , probeNodeState = nodeState + , probeNodeTipInfo = mNodeTipInfo + } + <- nodes + ] - explain :: NoTestnetReason -> String - explain DirMissing = "The directory " <> outputDir <> " does not exist." - explain NotReady = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." - explain NotATestnetDir = outputDir <> " is not a testnet output directory." + explain :: NoTestnetReason -> String + explain DirMissing = "The directory " <> outputDir <> " does not exist." + explain NotReady = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + explain NotATestnetDir = outputDir <> " is not a testnet output directory." - errors :: [String] - errors = - [ nodeName <> ": " <> err - | NodeProbeResult{probeNodeName = nodeName, probeError = Just err} <- nodes - ] + errors :: [String] + errors = + [ nodeName <> ": " <> err + | NodeProbeResult{probeNodeName = nodeName, probeError = Just err} <- nodes + ] -- | Render an array of strings as a table, where each string in the output list is a line. renderTable :: [[String]] -> [String] renderTable rows = - ("+" ++ concat ["-" ++ replicate width '-' ++ "-+" | width <- widths]) + ("+" ++ concat ["-" ++ replicate width '-' ++ "-+" | width <- widths]) : concatMap ( \row -> [ concat ("|" : [" " ++ pad width cell ++ " |" | (cell, width) <- zip row widths]) @@ -349,3 +331,4 @@ renderTable rows = pad :: Int -> String -> [Char] pad w s = s ++ replicate (w - length s) ' ' + From 5d9035eed1fbaaf4367bdddb18b9719f62147b30 Mon Sep 17 00:00:00 2001 From: Pablo Lamela Date: Tue, 27 Oct 2026 17:40:36 +0000 Subject: [PATCH 15/15] Address the latest review comments on the status command --- cardano-testnet/app/cardano-testnet.hs | 3 ++ cardano-testnet/src/Testnet/Status/Run.hs | 35 ++++++++----------- .../Cardano/Testnet/Test/Golden/Status.hs | 10 +++--- .../Cardano/Testnet/Test/Status.hs | 4 +-- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/cardano-testnet/app/cardano-testnet.hs b/cardano-testnet/app/cardano-testnet.hs index 53de472de9a..f1f9776dbb9 100644 --- a/cardano-testnet/app/cardano-testnet.hs +++ b/cardano-testnet/app/cardano-testnet.hs @@ -4,11 +4,14 @@ import Cardano.CLI.Environment (getEnvCli) import qualified Cardano.Crypto.Init as Crypto import qualified Options.Applicative as Opt +import System.IO (BufferMode (LineBuffering), hSetBuffering, stderr, stdout) import Parsers.Run (opts, pref, runTestnetCmd) main :: IO () main = do + hSetBuffering stdout LineBuffering + hSetBuffering stderr LineBuffering Crypto.cryptoInit envCli <- getEnvCli diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs index 697a09f401e..960434f812c 100644 --- a/cardano-testnet/src/Testnet/Status/Run.hs +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -51,7 +51,7 @@ import UnliftIO.Async (forConcurrently) -- and exit with 0 (running or degraded), 3 (stalled), 4 (stopped) or 5 (no -- testnet). When the manifest or the genesis file cannot be read, print the -- error on stderr and exit with 1. -runCheckStatusOptions :: CheckStatusOptions -> IO () +runCheckStatusOptions :: HasCallStack => CheckStatusOptions -> IO () runCheckStatusOptions CheckStatusOptions { testnetDir = dir @@ -106,19 +106,14 @@ checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = Manifes slotStart (SlotNo slot) = addUTCTime (fromIntegral slot * slotLength) (sgSystemStart genesis) -- | Check the status of one node in the testnet. -probeNode :: - -- | output directory - FilePath -> - -- | timeout, in seconds - Nat -> - -- | network magic - Word32 -> - -- | tip age limit - NominalDiffTime -> - -- | start time of a slot - (SlotNo -> UTCTime) -> - ManifestNode -> - IO NodeProbeResult +probeNode + :: FilePath -- ^ output directory + -> Nat -- ^ timeout, in seconds + -> Word32 -- ^ network magic + -> NominalDiffTime -- ^ tip age limit + -> (SlotNo -> UTCTime) -- ^ start time of a slot + -> ManifestNode + -> IO NodeProbeResult probeNode outputDir timeoutSeconds magic limit slotStart ManifestNode { mnodeName = nodeName @@ -177,13 +172,11 @@ classifyNetwork states -- | The rule for one node. The answer decides; the pid only tells "down" -- (the process is gone) from "unreachable" (alive, or unknown). -calcNodeState :: - -- | tip age limit - NominalDiffTime -> - -- | whether the node's process is alive, if known - Maybe Bool -> - NodeAnswer -> - NodeState +calcNodeState + :: NominalDiffTime -- ^ tip age limit + -> Maybe Bool -- ^ whether the node's process is alive, if known + -> NodeAnswer + -> NodeState calcNodeState limit _pidAlive (AnsweredTip tip) | tipAge tip <= limit = NodeOk | otherwise = NodeStalled diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs index 044abd66bb2..a7c393ca52f 100644 --- a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -15,7 +15,7 @@ import Cardano.Testnet.Test.Golden.Util (execDetailCardanoTestnet) import Prelude -import Data.Aeson (Value (..), eitherDecodeFileStrict, eitherDecodeStrict) +import Data.Aeson (Value (..), eitherDecodeStrict) import Data.Aeson.Encode.Pretty (encodePretty) import qualified Data.Aeson.KeyMap as KeyMap import Data.ByteString (ByteString) @@ -32,9 +32,8 @@ import Testnet.Manifest (ManifestNodeRole (..)) import Testnet.Status.Run (renderStatusReportHumanReadable) import Testnet.Status.Types -import Hedgehog (Property, evalIO, (===)) -import Hedgehog.Extras.Test.Base (assertWith, leftFail, leftFailM, moduleWorkspace, - propertyOnce) +import Hedgehog (Property, (===)) +import Hedgehog.Extras.Test.Base (assertWith, leftFail, moduleWorkspace, propertyOnce) import qualified Hedgehog.Extras.Test.File as H import Hedgehog.Extras.Test.Golden (diffVsGoldenFile) import qualified Hedgehog.Extras.Test.Process as H @@ -189,8 +188,7 @@ golden_StatusCommand = propertyOnce . moduleWorkspace "status-command" $ \dir -> (jsonCode, jsonOut, jsonErr) <- execDetailCardanoTestnet ["status", "--testnet-dir", missing, "--json"] jsonCode === ExitFailure 5 json <- leftFail (eitherDecodeStrict (Text.encodeUtf8 $ Text.pack jsonOut) :: Either String Value) - golden <- leftFailM . evalIO $ - (eitherDecodeFileStrict (base goldenFile "status_no_testnet.json") :: IO (Either String Value)) + golden <- H.readJsonFileOk $ base goldenFile "status_no_testnet.json" withoutCheckedAt json === withoutCheckedAt golden jsonErr === "" diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs index 9ba940a666f..b3c298082d2 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -48,7 +48,7 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul running <- H.evalIO $ checkStatus tmpDir 5 reportStatus running === NetworkRunning map probeNodeState (reportNodes running) === map (const NodeOk) (toList testnetNodes) - H.assert . isJust $ reportBestTip running + H.assertWith (reportBestTip running) isJust relay <- H.evalMaybe $ find (not . isTestnetNodeSpo) testnetNodes H.evalIO $ do @@ -60,7 +60,7 @@ hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaul reportStatus degraded === NetworkDegraded killed <- H.evalMaybe $ find ((== nodeName relay) . probeNodeName) (reportNodes degraded) probeNodeState killed === NodeDown - H.assert . isJust $ probeError killed + H.assertWith (probeError killed) isJust H.evalIO . forM_ (filter ((/= nodeName relay) . nodeName) $ toList testnetNodes) $ \node -> do hardKillProcess (nodeProcessHandle node)