Repository navigation
fix: handle LLM provider failures in validator voting and leader recovery #1424
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f389f55
a84e40a
c585cff
3ed7610
c7d37d7
548e2d2
3fd8d82
992fdf2
a40502f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,7 @@ | |
| DEFAULT_VALIDATORS_COUNT = 5 | ||
| DEFAULT_CONSENSUS_SLEEP_TIME = 5 | ||
| ACTIVATED_TRANSACTION_TIMEOUT = 900 | ||
| MAX_IDLE_REPLACEMENTS = 3 | ||
|
|
||
| import os | ||
| import asyncio | ||
|
|
@@ -60,6 +61,7 @@ | |
| from backend.consensus.types import ConsensusResult, ConsensusRound | ||
| from backend.consensus.utils import determine_consensus_from_votes | ||
| from backend.node.genvm import get_code_slot | ||
| from backend.node.genvm.error_codes import GenVMInternalError, GenVMErrorCode | ||
| from backend.node.base import Manager as GenVMManager | ||
|
|
||
| type NodeFactory = Callable[ | ||
|
|
@@ -2666,26 +2668,45 @@ def leader_timing_callback(step_name: str): | |
| context.transaction.hash, f"PROPOSING.LEADER.{step_name}" | ||
| ) | ||
|
|
||
| # Create a leader node for executing the transaction | ||
| leader_node = context.node_factory( | ||
| context.leader, | ||
| ExecutionMode.LEADER, | ||
| deepcopy(context.contract_snapshot), | ||
| None, | ||
| context.msg_handler, | ||
| context.contract_snapshot_factory, | ||
| context.validators_snapshot, | ||
| leader_timing_callback, | ||
| context.genvm_manager, | ||
| ) | ||
| # Execute leader with replacement on fatal infrastructure failures | ||
| for attempt in range(MAX_IDLE_REPLACEMENTS + 1): | ||
| leader_node = context.node_factory( | ||
| context.leader, | ||
| ExecutionMode.LEADER, | ||
| deepcopy(context.contract_snapshot), | ||
| None, | ||
| context.msg_handler, | ||
| context.contract_snapshot_factory, | ||
| context.validators_snapshot, | ||
| leader_timing_callback, | ||
| context.genvm_manager, | ||
| ) | ||
|
|
||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "PROPOSING.LEADER_NODE_CREATED" | ||
| ) | ||
| # Execute the transaction and obtain the leader receipt | ||
| context.consensus_data.leader_receipt = [ | ||
| await leader_node.exec_transaction(context.transaction) | ||
| ] | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, | ||
| f"PROPOSING.LEADER_NODE_CREATED.attempt_{attempt}", | ||
| ) | ||
| try: | ||
| context.consensus_data.leader_receipt = [ | ||
| await leader_node.exec_transaction(context.transaction) | ||
| ] | ||
| break # success | ||
| except GenVMInternalError as e: | ||
| if not e.is_fatal: | ||
| raise # non-fatal → propagate immediately | ||
| if not context.remaining_validators: | ||
| raise # pool empty → propagate | ||
| # Replace leader with next validator | ||
| context.leader = context.remaining_validators.pop(0) | ||
| else: | ||
| # All replacement attempts exhausted | ||
| raise GenVMInternalError( | ||
| message="Leader idle: all replacements exhausted", | ||
| error_code=GenVMErrorCode.LLM_NO_PROVIDER, | ||
| causes=["ALL_LEADERS_IDLE"], | ||
| is_fatal=True, | ||
| is_leader=True, | ||
| ) | ||
|
|
||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "PROPOSING.TRANSACTION_EXECUTED" | ||
|
|
@@ -2815,15 +2836,50 @@ def validator_timing_callback(step_name: str): | |
| # Execute the transaction with a semaphore to limit the number of concurrent validators | ||
| sem = asyncio.Semaphore(8) | ||
|
|
||
| async def run_single_validator(validator: Node, index: int) -> Receipt: | ||
| # Build replacement pool: all validators minus those already assigned | ||
| assigned_addresses: set[str] = set() | ||
| if context.leader.get("address"): | ||
| assigned_addresses.add(context.leader["address"]) | ||
| assigned_addresses.update(v["address"] for v in context.remaining_validators) | ||
| replacement_pool: list[dict] = [ | ||
| n.validator.to_dict() | ||
| for n in context.validators_snapshot.nodes | ||
| if n.validator.to_dict()["address"] not in assigned_addresses | ||
| ] | ||
| pool_lock = asyncio.Lock() | ||
|
|
||
| async def pop_replacement() -> dict | None: | ||
| async with pool_lock: | ||
| return replacement_pool.pop(0) if replacement_pool else None | ||
|
|
||
| def _is_fatal_error(receipt: Receipt) -> bool: | ||
| raw_error = (receipt.genvm_result or {}).get("raw_error") | ||
| return isinstance(raw_error, dict) and raw_error.get("fatal") is True | ||
|
|
||
| async def run_single_validator(validator_dict: dict, index: int) -> Receipt: | ||
| async with sem: | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, f"COMMITTING.VALIDATOR_{index}_START" | ||
| ) | ||
| result = await validator.exec_transaction(context.transaction) | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, f"COMMITTING.VALIDATOR_{index}_END" | ||
| ) | ||
| current = validator_dict | ||
| for attempt in range(MAX_IDLE_REPLACEMENTS + 1): | ||
| node = create_validator_node(context, current, index) | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, | ||
| f"COMMITTING.VALIDATOR_{index}_START" f".attempt_{attempt}", | ||
| ) | ||
| result = await node.exec_transaction(context.transaction) | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, | ||
| f"COMMITTING.VALIDATOR_{index}_END" f".attempt_{attempt}", | ||
| ) | ||
|
|
||
| if _is_fatal_error(result) and attempt < MAX_IDLE_REPLACEMENTS: | ||
| replacement = await pop_replacement() | ||
| if replacement is not None: | ||
| current = replacement | ||
| continue | ||
| break | ||
|
|
||
| if _is_fatal_error(result): | ||
| result.vote = Vote.IDLE | ||
| return result | ||
|
Comment on lines
+2855
to
2883
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: #!/bin/bash
# Check Vote enum definition for IDLE member
echo "=== Searching for Vote enum definition ==="
rg -n "class Vote" --type=py -A 20
echo -e "\n=== Checking for IDLE in Vote enum ==="
rg -n "IDLE" --type=py -B 2 -A 2 | head -50Repository: genlayerlabs/genlayer-studio Length of output: 4633 🏁 Script executed: #!/bin/bash
# Check GenVMInternalError and is_fatal in leader path
echo "=== GenVMInternalError exception definition ==="
rg -n "class GenVMInternalError" --type=py -A 10
echo -e "\n=== is_fatal attribute usage ==="
rg -n "is_fatal" --type=py -B 3 -A 3Repository: genlayerlabs/genlayer-studio Length of output: 11310 🏁 Script executed: #!/bin/bash
# Check raw_error and fatal in Node.exec_transaction
echo "=== Searching for raw_error and fatal in node base ==="
rg -n "raw_error" --type=py -B 2 -A 2
echo -e "\n=== genvm_result structure ==="
rg -n "genvm_result" --type=py -B 1 -A 1 | head -50Repository: genlayerlabs/genlayer-studio Length of output: 10474 🏁 Script executed: #!/bin/bash
# Verify the exact structure created at line 861
echo "=== backend/node/base.py around line 861 ==="
sed -n '850,870p' backend/node/base.py
echo -e "\n=== backend/node/base.py around line 917 ==="
sed -n '910,930p' backend/node/base.pyRepository: genlayerlabs/genlayer-studio Length of output: 1989 🏁 Script executed: #!/bin/bash
# Double-check what raw_error key name is used in tests for validation
echo "=== Test expectations for raw_error key ==="
rg -n 'raw_error.*fatal' --type=py -B 1 -A 1 | head -40Repository: genlayerlabs/genlayer-studio Length of output: 2716 Fix inconsistent raw_error key naming in GenVMInternalError handling.
When GenVMInternalError is caught during validator execution (backend/node/base.py:861), the receipt's raw_error is created with "raw_error": {"causes": e.causes, "is_fatal": e.is_fatal}However, both the leader path check (backend/node/base.py:917) and consensus layer validator replacement check (backend/consensus/base.py:2857) expect the if raw_error.get("fatal") is True:This causes the 🤖 Prompt for AI Agents |
||
|
|
||
| # Leader evaluates validation function | ||
|
|
@@ -2832,40 +2888,37 @@ async def run_single_validator(validator: Node, index: int) -> Receipt: | |
| and len(context.consensus_data.leader_receipt) == 1 | ||
| ) | ||
|
|
||
| # Create validator node for the leader | ||
| # Build list of validator dicts to run | ||
| if validation_by_leader: | ||
| context.validator_nodes = [ | ||
| create_validator_node(context, context.leader, 0) | ||
| ] | ||
| validators_to_run = [context.leader] + context.remaining_validators | ||
| else: | ||
| context.validator_nodes = [] | ||
| validators_to_run = list(context.remaining_validators) | ||
|
|
||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "COMMITTING.LEADER_NODE_CREATED" | ||
| ) | ||
| # Create validator nodes for each validator | ||
| start_index = 1 if validation_by_leader else 0 | ||
| context.validator_nodes.extend( | ||
| [ | ||
| create_validator_node(context, validator, start_index + index) | ||
| for index, validator in enumerate(context.remaining_validators) | ||
| ] | ||
| ) | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "COMMITTING.VALIDATOR_NODES_CREATED" | ||
| context.transaction.hash, "COMMITTING.VALIDATORS_PREPARED" | ||
| ) | ||
|
|
||
| # Execute the transaction on each validator node and gather the results | ||
| # Execute the transaction on each validator and gather the results | ||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "COMMITTING.VALIDATORS_EXECUTION_START" | ||
| ) | ||
|
|
||
| validation_tasks = [ | ||
| run_single_validator(validator, index) | ||
| for index, validator in enumerate(context.validator_nodes) | ||
| run_single_validator(validator_dict, index) | ||
| for index, validator_dict in enumerate(validators_to_run) | ||
| ] | ||
| context.validation_results = await asyncio.gather(*validation_tasks) | ||
|
|
||
| # If all validators voted IDLE, infrastructure is systemically broken | ||
| if all(r.vote == Vote.IDLE for r in context.validation_results): | ||
| raise GenVMInternalError( | ||
| message="All validators idle after replacements", | ||
| error_code=GenVMErrorCode.LLM_NO_PROVIDER, | ||
| causes=["ALL_VALIDATORS_IDLE"], | ||
| is_fatal=True, | ||
| is_leader=True, | ||
| ) | ||
|
|
||
| context.transactions_processor.add_state_timestamp( | ||
| context.transaction.hash, "COMMITTING.VALIDATORS_EXECUTION_END" | ||
| ) | ||
|
|
@@ -2925,7 +2978,7 @@ async def handle(self, context): | |
| # Process each validation result and update the context | ||
| for i, validation_result in enumerate(context.validation_results): | ||
| # Store the vote from each validator node | ||
| context.votes[context.validator_nodes[i].address] = ( | ||
| context.votes[validation_result.node_config["address"]] = ( | ||
| validation_result.vote.value | ||
| ) | ||
|
|
||
|
|
@@ -2934,14 +2987,21 @@ async def handle(self, context): | |
| consensus_result = determine_consensus_from_votes(votes_list) | ||
|
|
||
| # Send event in rollup to communicate the votes are revealed | ||
| # Vote.IDLE maps to Vote.TIMEOUT for on-chain events (Solidity | ||
| # VoteType enum doesn't have IDLE) | ||
| for i, validation_result in enumerate(context.validation_results): | ||
| last_vote = i == len(context.validation_results) - 1 | ||
| chain_vote = ( | ||
| Vote.TIMEOUT | ||
| if validation_result.vote == Vote.IDLE | ||
| else validation_result.vote | ||
| ) | ||
| context.consensus_service.emit_transaction_event( | ||
| "emitVoteRevealed", | ||
| validation_result.node_config, | ||
| context.transaction.hash, | ||
| validation_result.node_config["address"], | ||
| int(validation_result.vote), | ||
| int(chain_vote), | ||
| last_vote, | ||
| int(consensus_result) if last_vote else int(ConsensusResult.IDLE), | ||
| ) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Select replacement validators via VRF instead of snapshot order.
The replacement pool is currently built in snapshot order, which bypasses the VRF-based rotation requirement for consensus.
🔁 VRF-based replacement selection
As per coding guidelines, "backend/consensus/**/*.py: Implement validator rotation using VRF (Verifiable Random Function) in the consensus system".
🤖 Prompt for AI Agents