Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions backend/protocol_rpc/health.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,9 @@ async def _run_health_checks() -> None:
"seconds_since_consensus_progress": consensus_health.get(
"seconds_since_consensus_progress"
),
"no_progress_check_error": consensus_health.get(
"no_progress_check_error", False
),
"active_workers": consensus_health.get("active_workers", 0),
"status": consensus_status,
}
Expand Down Expand Up @@ -870,12 +873,19 @@ def _query_consensus():
exc,
)

no_recent_progress = should_scan_progress and (
not last_progress_epoch
or (
seconds_since_consensus_progress is not None
and seconds_since_consensus_progress
> no_progress_window_seconds
# The progress scan is an alert-quality check, not a liveness
# requirement. If it times out on a large table, surface that
# as check_error but do not assert a consensus outage.
no_recent_progress = (
should_scan_progress
and not no_progress_check_error
and (
not last_progress_epoch
or (
seconds_since_consensus_progress is not None
and seconds_since_consensus_progress
> no_progress_window_seconds
)
)
)
no_consensus_progress = (
Expand Down
44 changes: 44 additions & 0 deletions tests/db-sqlalchemy/test_health_orphan_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,50 @@ def fail_on_progress_scan(
assert result["no_progress_backlog_count"] == 0


@pytest.mark.asyncio
async def test_no_progress_scan_error_does_not_assert_outage(
engine: Engine, monkeypatch: pytest.MonkeyPatch
):
"""If the optional JSON history scan times out, surface the check error
but don't turn an unknown progress signal into a consensus outage."""
monkeypatch.setenv("HEALTH_NO_PROGRESS_WINDOW_MINUTES", "10")
monkeypatch.setenv("HEALTH_NO_PROGRESS_MIN_BACKLOG", "3")

Session_ = sessionmaker(bind=engine, expire_on_commit=False)
now = datetime.now(timezone.utc)

with Session_() as s:
for i in range(3):
_insert_tx(
s,
tx_hash=f"0xd0{i:062x}",
to_address="0x" + "d0" * 20,
status="PENDING",
nonce=i,
created_at=now - timedelta(minutes=20 + i),
)
s.commit()

def fail_on_progress_scan(
conn, cursor, statement, parameters, context, executemany
):
if "current_monitoring" in statement:
raise RuntimeError("simulated progress scan timeout")

event.listen(engine, "before_cursor_execute", fail_on_progress_scan)
try:
from backend.protocol_rpc import health as health_module

result = await health_module._check_consensus_health()
finally:
event.remove(engine, "before_cursor_execute", fail_on_progress_scan)

assert result["no_progress_check_error"] is True
assert result["no_consensus_progress"] is False
assert result["no_progress_backlog_count"] == 3
assert result["status"] == "healthy"


@pytest.mark.asyncio
async def test_recent_consensus_progress_suppresses_no_progress_alert(
engine: Engine, monkeypatch: pytest.MonkeyPatch
Expand Down
Loading