Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 131 additions & 39 deletions backend/protocol_rpc/health.py
Original file line number Diff line number Diff line change
Expand Up @@ -485,66 +485,158 @@ def _db_ping():


async def _check_consensus_health() -> Dict[str, Any]:
"""Check consensus system status."""
from datetime import datetime, timedelta, timezone
from backend.database_handler.models import Transactions
"""Check consensus system status.

"Orphaned transactions" here means *contracts whose head-of-queue tx
isn't making progress* — not "lots of pending txs". A long queue is
not a problem if the head is moving; the head being stuck IS a
problem regardless of queue depth.

A contract's head is the oldest non-final tx for that contract. The
head is "stuck" when:
- it was created more than `head_stuck_after_minutes` ago
(enough time to expect *some* progress), AND
- no tx for that contract has a fresh `blocked_at` within the
`recent_activity_window_minutes` window (i.e. no worker is
currently doing anything for the contract).

`active_workers` is derived from unexpired worker claims so it
correctly reflects workers processing OLD txs (the previous "txs
created in last 1h with worker_id" filter falsely reported zero
workers when traffic was bursty).

Note on the "claim window": workers do NOT heartbeat `blocked_at`
during execution — it's set once at claim time, cleared on
completion. So "fresh blocked_at" must mean "claim not yet
expired", aligned with TRANSACTION_TIMEOUT_MINUTES, not "claimed
within the last few minutes" (which would falsely flag a long
consensus round as inactive).
"""
import os

from backend.database_handler.session_factory import get_database_manager

HEAD_STUCK_AFTER_MINUTES = int(
os.environ.get("HEALTH_HEAD_STUCK_AFTER_MINUTES", "15")
)
# Aligned with the worker's claim timeout. While blocked_at sits
# within this window, a worker still legitimately owns the tx and
# we must not call its contract "stuck". Past this window, recovery
# would also reset the claim. Default 30 matches docker-compose /
# prod manifests.
CLAIM_WINDOW_MINUTES = int(os.environ.get("TRANSACTION_TIMEOUT_MINUTES", "30"))
# Match the dashboard alert threshold so consensus.status and the
# top-level "issues" tag agree on what counts as degraded.
DEGRADED_AT_STUCK_HEADS = int(os.environ.get("HEALTH_DEGRADED_AT_STUCK_HEADS", "3"))
Comment on lines +528 to +530

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

DEGRADED_AT_STUCK_HEADS doesn’t fully align /health yet.

This new threshold only affects _check_consensus_health(). _run_health_checks() still hard-codes >= 3 when it downgrades the top-level /health payload, so any env override will make services.consensus.status disagree with top-level status/issues.

Suggested fix
-        elif consensus_health.get("total_orphaned_transactions", 0) >= 3:
+        elif consensus_status == "degraded":
             if overall_status == "healthy":
                 overall_status = "degraded"
             issues.append("orphaned_transactions")
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@backend/protocol_rpc/health.py` around lines 528 - 530, _top-level health
downgrade uses a hard-coded ">= 3" while you introduced DEGRADED_AT_STUCK_HEADS,
causing services.consensus.status (from _check_consensus_health) to disagree
with the top-level /health status and issues; update _run_health_checks() to use
the DEGRADED_AT_STUCK_HEADS constant instead of the literal 3 so the same
threshold drives both the consensus check and the top-level status/issues logic
(look for DEGRADED_AT_STUCK_HEADS, _check_consensus_health(), and
_run_health_checks() to make the replacement and ensure the comparison semantics
remain the same).


# Statuses considered "in flight". Includes the *_TIMEOUT variants
# because finalization and appeal claim paths process them. Must
# match the head-CTE filter and the in-flight-count query for
# consistency.
IN_FLIGHT_STATUSES_SQL = (
"'ACTIVATED','PROPOSING','COMMITTING','REVEALING',"
"'ACCEPTED','UNDETERMINED','LEADER_TIMEOUT','VALIDATORS_TIMEOUT'"
)
Comment on lines +528 to +539

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

PENDING heads are currently invisible to the stuck-head detector.

The docstring here defines the head as the oldest non-final tx, but the shared status set excludes PENDING. That means a contract whose oldest tx has been sitting in PENDING past the stuck threshold with no claim will never contribute to total_orphaned_transactions, which misses one real head-of-queue stall mode. If you want total_processing_transactions to stay narrower than the head detector, this probably needs separate status sets instead of one shared constant.

Also applies to: 579-600, 615-620

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@backend/protocol_rpc/health.py` around lines 528 - 539, The stuck-head
detector currently omits PENDING because IN_FLIGHT_STATUSES_SQL excludes it, so
oldest non-final txes in PENDING don't count toward total_orphaned_transactions;
fix by introducing a separate status constant (e.g. STUCK_HEAD_STATUSES_SQL or
HEAD_NON_FINAL_STATUSES) that includes 'PENDING' while keeping
IN_FLIGHT_STATUSES_SQL as-is if you want processing counts narrower, then update
the head-CTE filter, the stuck-head query (where total_orphaned_transactions is
computed), and any references at the spots noted (including uses around lines
referenced and functions/queries that reference IN_FLIGHT_STATUSES_SQL) to use
the new STUCK_HEAD_STATUSES_SQL so the detector treats PENDING as non-final
head-of-queue.


try:
if not _rpc_router_ref:
return {"status": "not_initialized", "error": "RPC router not available"}

db_manager = get_database_manager()

def _query_consensus():
now = datetime.now(timezone.utc)
recent_threshold = now - timedelta(hours=1)

from sqlalchemy import select, distinct, and_, text

# Get active worker IDs
with db_manager.engine.connect() as worker_conn:
worker_query = select(distinct(Transactions.worker_id)).where(
and_(
Transactions.worker_id.isnot(None),
Transactions.created_at > recent_threshold,
)
)
worker_result = worker_conn.execute(worker_query)
active_workers = {row[0] for row in worker_result if row[0]}
from sqlalchemy import text

# Get transaction statistics
with db_manager.engine.connect() as conn:
query = text(
"""
SELECT
COUNT(*) FILTER (WHERE status IN ('ACTIVATED', 'PROPOSING', 'COMMITTING', 'REVEALING', 'ACCEPTED', 'UNDETERMINED')) as processing_count,
COUNT(*) FILTER (WHERE worker_id IS NOT NULL AND status IN ('PENDING', 'ACTIVATED', 'PROPOSING', 'COMMITTING', 'REVEALING', 'ACCEPTED', 'UNDETERMINED')) as blocked_count
FROM transactions
WHERE to_address IS NOT NULL
"""
)
result = conn.execute(query)
row = result.fetchone()

total_processing = row.processing_count if row else 0
total_blocked = row.blocked_count if row else 0
# Active workers: distinct worker_ids with an unexpired
# claim. Catches workers processing old txs that the
# previous "created_at > 1h ago" filter incorrectly
# excluded.
active_workers_row = conn.execute(
text(
"""
SELECT COUNT(DISTINCT worker_id) AS n
FROM transactions
WHERE worker_id IS NOT NULL
AND blocked_at IS NOT NULL
AND blocked_at > NOW() - make_interval(mins => :claim_window)
"""
),
{"claim_window": CLAIM_WINDOW_MINUTES},
).fetchone()
active_workers_count = active_workers_row.n if active_workers_row else 0

# Stuck heads: per contract, the oldest in-flight tx
# whose contract has no unexpired worker claim AND head
# is old enough to expect progress. Returns the count
# of AFFECTED CONTRACTS, not the count of queued txs
# behind them — those are not the problem, the head is.
# Tie-break on hash so debug output (when we surface
# the heads) is deterministic.
stuck_row = conn.execute(
text(
f"""
WITH heads AS (
SELECT DISTINCT ON (to_address)
to_address,
hash,
status,
created_at
FROM transactions
WHERE status IN ({IN_FLIGHT_STATUSES_SQL})
AND to_address IS NOT NULL
ORDER BY to_address, created_at ASC, hash ASC
)
SELECT COUNT(*) AS stuck_heads
FROM heads h
WHERE h.created_at < NOW() - make_interval(mins => :head_stuck_minutes)
AND NOT EXISTS (
SELECT 1
FROM transactions t2
WHERE t2.to_address = h.to_address
AND t2.status IN ({IN_FLIGHT_STATUSES_SQL})
AND t2.blocked_at IS NOT NULL
AND t2.blocked_at > NOW() - make_interval(mins => :claim_window)
)
"""
),
{
"head_stuck_minutes": HEAD_STUCK_AFTER_MINUTES,
"claim_window": CLAIM_WINDOW_MINUTES,
},
).fetchone()
stuck_head_contracts = stuck_row.stuck_heads if stuck_row else 0

total_orphaned = 0
if total_blocked > 0 and len(active_workers) == 0:
total_orphaned = total_blocked
# Total in-flight (non-final) tx count, for context.
# Same status set as the head CTE so the metrics are
# consistent.
processing_row = conn.execute(
text(
f"""
SELECT COUNT(*) AS n
FROM transactions
WHERE status IN ({IN_FLIGHT_STATUSES_SQL})
AND to_address IS NOT NULL
"""
)
).fetchone()
total_processing = processing_row.n if processing_row else 0

status = (
"healthy"
if total_processing < 100 and total_orphaned == 0
if stuck_head_contracts < DEGRADED_AT_STUCK_HEADS
else "degraded"
)

return {
"status": status,
"total_processing_transactions": total_processing,
"total_orphaned_transactions": total_orphaned,
"active_workers": len(active_workers),
# Field name preserved for backwards-compat with the
# external dashboard. Semantics changed: now counts
# CONTRACTS whose head-of-queue is stuck, not raw
# blocked txs. A long queue with a moving head reads 0.
"total_orphaned_transactions": stuck_head_contracts,
"active_workers": active_workers_count,
}

return await asyncio.to_thread(_query_consensus)
Expand Down
Loading
Loading