From 78881c74bcc4bb91df8450098f6ce0e27151a497 Mon Sep 17 00:00:00 2001 From: Edgars Date: Mon, 25 May 2026 00:49:26 +0100 Subject: [PATCH] fix: avoid false no-progress alerts on scan timeout --- backend/protocol_rpc/health.py | 22 +++++++--- .../test_health_orphan_detection.py | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/backend/protocol_rpc/health.py b/backend/protocol_rpc/health.py index aa28bacb6..534b1cd15 100644 --- a/backend/protocol_rpc/health.py +++ b/backend/protocol_rpc/health.py @@ -319,6 +319,9 @@ async def _run_health_checks() -> None: "seconds_since_consensus_progress": consensus_health.get( "seconds_since_consensus_progress" ), + "no_progress_check_error": consensus_health.get( + "no_progress_check_error", False + ), "active_workers": consensus_health.get("active_workers", 0), "status": consensus_status, } @@ -870,12 +873,19 @@ def _query_consensus(): exc, ) - no_recent_progress = should_scan_progress and ( - not last_progress_epoch - or ( - seconds_since_consensus_progress is not None - and seconds_since_consensus_progress - > no_progress_window_seconds + # The progress scan is an alert-quality check, not a liveness + # requirement. If it times out on a large table, surface that + # as check_error but do not assert a consensus outage. + no_recent_progress = ( + should_scan_progress + and not no_progress_check_error + and ( + not last_progress_epoch + or ( + seconds_since_consensus_progress is not None + and seconds_since_consensus_progress + > no_progress_window_seconds + ) ) ) no_consensus_progress = ( diff --git a/tests/db-sqlalchemy/test_health_orphan_detection.py b/tests/db-sqlalchemy/test_health_orphan_detection.py index 40d182be1..437afe7dd 100644 --- a/tests/db-sqlalchemy/test_health_orphan_detection.py +++ b/tests/db-sqlalchemy/test_health_orphan_detection.py @@ -688,6 +688,50 @@ def fail_on_progress_scan( assert result["no_progress_backlog_count"] == 0 +@pytest.mark.asyncio +async def test_no_progress_scan_error_does_not_assert_outage( + engine: Engine, monkeypatch: pytest.MonkeyPatch +): + """If the optional JSON history scan times out, surface the check error + but don't turn an unknown progress signal into a consensus outage.""" + monkeypatch.setenv("HEALTH_NO_PROGRESS_WINDOW_MINUTES", "10") + monkeypatch.setenv("HEALTH_NO_PROGRESS_MIN_BACKLOG", "3") + + Session_ = sessionmaker(bind=engine, expire_on_commit=False) + now = datetime.now(timezone.utc) + + with Session_() as s: + for i in range(3): + _insert_tx( + s, + tx_hash=f"0xd0{i:062x}", + to_address="0x" + "d0" * 20, + status="PENDING", + nonce=i, + created_at=now - timedelta(minutes=20 + i), + ) + s.commit() + + def fail_on_progress_scan( + conn, cursor, statement, parameters, context, executemany + ): + if "current_monitoring" in statement: + raise RuntimeError("simulated progress scan timeout") + + event.listen(engine, "before_cursor_execute", fail_on_progress_scan) + try: + from backend.protocol_rpc import health as health_module + + result = await health_module._check_consensus_health() + finally: + event.remove(engine, "before_cursor_execute", fail_on_progress_scan) + + assert result["no_progress_check_error"] is True + assert result["no_consensus_progress"] is False + assert result["no_progress_backlog_count"] == 3 + assert result["status"] == "healthy" + + @pytest.mark.asyncio async def test_recent_consensus_progress_suppresses_no_progress_alert( engine: Engine, monkeypatch: pytest.MonkeyPatch