feat: improve monitoring script and add bolt task

Gemfile

@@ -22,9 +22,12 @@ group :development do
   gem "racc", '~> 1.4.0',                        require: false if Gem::Requirement.create(['>= 2.7.0', '< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "deep_merge", '~> 1.2.2',                  require: false
   gem "voxpupuli-puppet-lint-plugins", '~> 5.0', require: false
-  gem "facterdb", '~> 2.1',                      require: false
+  gem "facterdb", '~> 2.1',                      require: false if Gem::Requirement.create(['< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "facterdb", '~> 3.0',                      require: false if Gem::Requirement.create(['>= 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "metadata-json-lint", '~> 4.0',            require: false
-  gem "rspec-puppet-facts", '~> 4.0',            require: false
+  gem "json-schema", '< 5.1.1',                  require: false
+  gem "rspec-puppet-facts", '~> 4.0',            require: false if Gem::Requirement.create(['< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "rspec-puppet-facts", '~> 5.0',            require: false if Gem::Requirement.create(['>= 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "dependency_checker", '~> 1.0.0',          require: false
   gem "parallel_tests", '= 3.12.1',              require: false
   gem "pry", '~> 0.10',                          require: false
@@ -34,7 +37,6 @@ group :development do
   gem "rubocop-performance", '= 1.16.0',         require: false
   gem "rubocop-rspec", '= 2.19.0',               require: false
   gem "rb-readline", '= 0.5.5',                  require: false, platforms: [:mswin, :mingw, :x64_mingw]
-  gem "rexml", '>= 3.0.0', '< 3.2.7',            require: false
 end
 group :development, :release_prep do
   gem "puppet-strings", '~> 4.0',         require: false

README.md

@@ -114,6 +114,18 @@ class { 'prosa::processors':
 Since processors have different configurations, `proc_settings` is provided as a `Hash` to accommodate all necessary configuration options.
 To determine which configurations to specify, refer to the documentation for the corresponding processor.
 
+### Tasks
+
+The module provides [Bolt](https://www.puppet.com/docs/bolt/latest/bolt.html) tasks to query ProSA instances:
+
+- `prosa::version`: returns the versions of ProSA and its processors.
+- `prosa::status`: returns the readiness of ProSA from the `prosa_ready` metric, and fails if ProSA is not ready.
+  It requires the monitoring script, deployed when a local Prometheus endpoint is configured in `observability`.
+
+```shell
+bolt task run prosa::status --targets <targets>
+```
+
 ## Reference
 
 For information on classes, types and functions see the [REFERENCE.md](./REFERENCE.md)

REFERENCE.md

@@ -25,6 +25,11 @@ Use the `prosa::processors` defined type if you want to create processors
 
 * [`ProSA::TelemetryLevel`](#ProSA--TelemetryLevel): A string that conforms to the ProSA `TelemetryLevel` syntax.
 
+### Tasks
+
+* [`status`](#status): Checks whether ProSA is ready to serve requests, using the prosa_ready metric. Fails if ProSA is not ready.
+* [`version`](#version): Gets the ProSA components versions
+
 ## Classes
 
 ### <a name="prosa"></a>`prosa`
@@ -395,3 +400,25 @@ ProSA accept non case sensitive levels (like this type).
 
 Alias of `Pattern[/\A(?i:error|warn|info|debug|trace|off)\Z/]`
 
+## Tasks
+
+### <a name="status"></a>`status`
+
+Checks whether ProSA is ready to serve requests, using the prosa_ready metric. Fails if ProSA is not ready.
+
+**Supports noop?** false
+
+#### Parameters
+
+##### `monitor_path`
+
+Data type: `Optional[String[1]]`
+
+Path of the ProSA monitoring script (default: /usr/local/bin/prosa-monitor)
+
+### <a name="version"></a>`version`
+
+Gets the ProSA components versions
+
+**Supports noop?** false
+

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "worldline-prosa",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "author": "Worldline",
   "summary": "Installs, configures, and manages ProSA.",
   "license": "LGPL-3.0",
@@ -10,7 +10,7 @@
   "dependencies": [
     {
       "name": "puppetlabs/stdlib",
-      "version_requirement": ">= 4.13.1 < 10.0.0"
+      "version_requirement": ">= 4.13.1 < 11.0.0"
     }
   ],
   "operatingsystem_support": [
@@ -52,6 +52,6 @@
     "payments"
   ],
   "pdk-version": "3.4.0",
-  "template-url": "pdk-default#3.4.0",
-  "template-ref": "tags/3.4.0-0-gd3cc13f"
+  "template-url": "https://github.com/puppetlabs/pdk-templates#3.4.0.3",
+  "template-ref": "tags/3.4.0.3-0-g8fb22fc"
 }

spec/classes/prosa_monitor_spec.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'open3'
+require 'socket'
+require 'tmpdir'
+
+PYTHON = `command -v python3 2>/dev/null`.strip
+PYTHON_VERSION = PYTHON.empty? ? [0, 0] : `#{PYTHON} -c 'import sys; print(*sys.version_info[:2])'`.split.map(&:to_i)
+
+METRICS_READY = <<~METRICS
+  prosa_ready{service_name="prosa",host_name="h1"} 1
+  prosa_main_ram{type="physical",service_name="prosa",host_name="h1"} 1048576
+  prosa_processors{type="node",id="1",title="stub",service_name="prosa",host_name="h1"} 0
+METRICS
+
+describe 'prosa' do
+  let(:facts) { on_supported_os.values.first }
+  let(:params) do
+    {
+      observability: {
+        'metrics' => { 'prometheus' => { 'endpoint' => '19090' } },
+        'traces' => { 'stdout' => { 'level' => 'info' } },
+        'logs' => { 'stdout' => { 'level' => 'info' } },
+      },
+    }
+  end
+
+  let(:script) { catalogue.resource('File', '/usr/local/bin/prosa-monitor')[:content] }
+
+  let(:dir) { Dir.mktmpdir('prosa-monitor-spec') }
+
+  before(:each) { File.write(File.join(dir, 'prosa-monitor'), script) }
+
+  after(:each) { FileUtils.remove_entry(dir) }
+
+  def run_monitor(*args)
+    Open3.capture3(PYTHON, '-I', File.join(dir, 'prosa-monitor'), *args)
+  end
+
+  # Serves `body` at /metrics on a free local port while the block runs.
+  def with_metrics(body)
+    root = File.join(dir, 'www')
+    Dir.mkdir(root)
+    File.write(File.join(root, 'metrics'), body)
+    port = TCPServer.open('127.0.0.1', 0) { |s| s.addr[1] }
+    pid = Process.spawn(PYTHON, '-I', '-m', 'http.server', port.to_s, '--bind', '127.0.0.1', '--directory', root,
+                        out: File::NULL, err: File::NULL)
+    begin
+      50.times do
+        TCPSocket.new('127.0.0.1', port).close
+        break
+      rescue Errno::ECONNREFUSED
+        sleep 0.1
+      end
+      yield "http://127.0.0.1:#{port}/metrics"
+    ensure
+      Process.kill('TERM', pid)
+      Process.wait(pid)
+    end
+  end
+
+  context 'prosa-monitor script', if: (PYTHON_VERSION <=> [3, 8]) >= 0 do
+    # The script must keep running on the oldest supported python3 (3.6 on RHEL 8).
+    it 'only uses Python 3.6 syntax' do
+      _out, err, status = Open3.capture3(
+        PYTHON, '-I', '-c',
+        'import ast, sys; ast.parse(open(sys.argv[1]).read(), feature_version=(3, 6))',
+        File.join(dir, 'prosa-monitor')
+      )
+      expect(status.success?).to be(true), err
+    end
+
+    it 'reports a ready instance' do
+      with_metrics(METRICS_READY) do |url|
+        out, _err, status = run_monitor('--check', url)
+        expect([out.strip, status.exitstatus]).to eq(['ready', 0])
+
+        out, _err, status = run_monitor('--no-color', url)
+        expect(out).to include('Ready: yes')
+        expect(status.exitstatus).to eq(0)
+      end
+    end
+
+    it 'reports a not ready instance' do
+      with_metrics(METRICS_READY.sub('} 1', '} 0')) do |url|
+        out, _err, status = run_monitor('--check', url)
+        expect([out.strip, status.exitstatus]).to eq(['not ready', 1])
+      end
+    end
+
+    it 'fails when the prosa_ready metric is absent' do
+      with_metrics(METRICS_READY.lines.reject { |l| l.start_with?('prosa_ready') }.join) do |url|
+        _out, err, status = run_monitor('--check', url)
+        expect(status.exitstatus).to eq(2)
+        expect(err).to include('no prosa_ready metric')
+      end
+    end
+
+    it 'fails when the endpoint is unreachable' do
+      port = TCPServer.open('127.0.0.1', 0) { |s| s.addr[1] }
+      _out, _err, status = run_monitor('--check', "http://127.0.0.1:#{port}/metrics")
+      expect(status.exitstatus).to eq(2)
+    end
+
+    it 'fails when the connection is reset' do
+      server = TCPServer.new('127.0.0.1', 0)
+      thread = Thread.new do
+        client = server.accept
+        client.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, [1, 0].pack('ii'))
+        client.readpartial(1024)
+        client.close
+      end
+      _out, err, status = run_monitor('--check', "http://127.0.0.1:#{server.addr[1]}/metrics")
+      thread.join
+      server.close
+      expect(status.exitstatus).to eq(2)
+      expect(err).not_to include('Traceback')
+    end
+
+    it 'rejects a non positive watch interval' do
+      _out, err, status = run_monitor('--watch', '0')
+      expect(status.exitstatus).to eq(2)
+      expect(err).not_to include('Traceback')
+    end
+  end
+end

spec/tasks/status_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'json'
+require 'open3'
+require 'rbconfig'
+require 'tmpdir'
+
+describe 'prosa::status task' do
+  let(:task) { File.expand_path('../../tasks/status.rb', __dir__) }
+  let(:dir) { Dir.mktmpdir('prosa-status-spec') }
+
+  after(:each) { FileUtils.remove_entry(dir) }
+
+  # Stub of `prosa-monitor --check` that prints `out` and exits with `code`.
+  def monitor_stub(out, code)
+    path = File.join(dir, 'prosa-monitor')
+    File.write(path, "#!/bin/sh\necho '#{out}'\nexit #{code}\n")
+    File.chmod(0o755, path)
+    path
+  end
+
+  def run_task(params)
+    out, _err, status = Open3.capture3(RbConfig.ruby, task, stdin_data: params.to_json)
+    [JSON.parse(out), status.exitstatus]
+  end
+
+  it 'reports a ready instance' do
+    result, code = run_task(monitor_path: monitor_stub('ready', 0))
+    expect([result['status'], code]).to eq(['ready', 0])
+  end
+
+  it 'reports a not ready instance' do
+    result, code = run_task(monitor_path: monitor_stub('not ready', 1))
+    expect([result['status'], code]).to eq(['not_ready', 1])
+  end
+
+  it 'reports the monitor error' do
+    result, code = run_task(monitor_path: monitor_stub('no prosa_ready metric', 2))
+    expect([result['status'], result['error'], code]).to eq(['error', 'no prosa_ready metric', 2])
+  end
+
+  it 'reports a monitor crash as an error' do
+    result, code = run_task(monitor_path: monitor_stub('Traceback', 1))
+    expect([result['status'], code]).to eq(['error', 2])
+  end
+
+  it 'fails when the monitoring script is missing' do
+    result, code = run_task(monitor_path: File.join(dir, 'missing'))
+    expect([result['status'], code]).to eq(['error', 2])
+    expect(result['error']).to include('not found')
+  end
+end

tasks/status.json

@@ -0,0 +1,10 @@
+{
+  "description": "Checks whether ProSA is ready to serve requests, using the prosa_ready metric. Fails if ProSA is not ready.",
+  "input_method": "stdin",
+  "parameters": {
+    "monitor_path": {
+      "description": "Path of the ProSA monitoring script (default: /usr/local/bin/prosa-monitor)",
+      "type": "Optional[String[1]]"
+    }
+  }
+}

tasks/status.rb

@@ -0,0 +1,50 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'socket'
+require 'time'
+
+DEFAULT_MONITOR_PATH = '/usr/local/bin/prosa-monitor'
+
+# Exit codes of `prosa-monitor --check`
+EXIT_READY = 0
+EXIT_NOT_READY = 1
+
+input = $stdin.read
+params = input.strip.empty? ? {} : JSON.parse(input)
+monitor_path = params['monitor_path'] || DEFAULT_MONITOR_PATH
+
+result = {
+  hostname: Socket.gethostname,
+  timestamp: Time.now.utc.iso8601,
+  monitor_path: monitor_path,
+}
+exit_code = 2
+
+begin
+  if File.executable?(monitor_path)
+    stdout, stderr, status = Open3.capture3(monitor_path, '--check')
+    state = [status.exitstatus, stdout.strip]
+    if state == [EXIT_READY, 'ready']
+      result[:status] = 'ready'
+      exit_code = EXIT_READY
+    elsif state == [EXIT_NOT_READY, 'not ready']
+      result[:status] = 'not_ready'
+      exit_code = EXIT_NOT_READY
+    else
+      result[:status] = 'error'
+      result[:error] = stderr.strip.empty? ? stdout.strip : stderr.strip
+    end
+  else
+    result[:status] = 'error'
+    result[:error] = "Monitoring script not found or not executable: #{monitor_path}"
+  end
+rescue StandardError => e
+  result[:status] = 'error'
+  result[:error] = e.message
+end
+
+puts JSON.pretty_generate(result)
+exit exit_code

tasks/version.json

@@ -0,0 +1,5 @@
+{
+  "description": "Gets the ProSA components versions",
+  "input_method": "stdin",
+  "parameters": {}
+}