Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 109 additions & 49 deletions backend/consensus/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
DEFAULT_VALIDATORS_COUNT = 5
DEFAULT_CONSENSUS_SLEEP_TIME = 5
ACTIVATED_TRANSACTION_TIMEOUT = 900
MAX_IDLE_REPLACEMENTS = 3

import os
import asyncio
Expand Down Expand Up @@ -60,6 +61,7 @@
from backend.consensus.types import ConsensusResult, ConsensusRound
from backend.consensus.utils import determine_consensus_from_votes
from backend.node.genvm import get_code_slot
from backend.node.genvm.error_codes import GenVMInternalError, GenVMErrorCode
from backend.node.base import Manager as GenVMManager

type NodeFactory = Callable[
Expand Down Expand Up @@ -2666,26 +2668,45 @@ def leader_timing_callback(step_name: str):
context.transaction.hash, f"PROPOSING.LEADER.{step_name}"
)

# Create a leader node for executing the transaction
leader_node = context.node_factory(
context.leader,
ExecutionMode.LEADER,
deepcopy(context.contract_snapshot),
None,
context.msg_handler,
context.contract_snapshot_factory,
context.validators_snapshot,
leader_timing_callback,
context.genvm_manager,
)
# Execute leader with replacement on fatal infrastructure failures
for attempt in range(MAX_IDLE_REPLACEMENTS + 1):
leader_node = context.node_factory(
context.leader,
ExecutionMode.LEADER,
deepcopy(context.contract_snapshot),
None,
context.msg_handler,
context.contract_snapshot_factory,
context.validators_snapshot,
leader_timing_callback,
context.genvm_manager,
)

context.transactions_processor.add_state_timestamp(
context.transaction.hash, "PROPOSING.LEADER_NODE_CREATED"
)
# Execute the transaction and obtain the leader receipt
context.consensus_data.leader_receipt = [
await leader_node.exec_transaction(context.transaction)
]
context.transactions_processor.add_state_timestamp(
context.transaction.hash,
f"PROPOSING.LEADER_NODE_CREATED.attempt_{attempt}",
)
try:
context.consensus_data.leader_receipt = [
await leader_node.exec_transaction(context.transaction)
]
break # success
except GenVMInternalError as e:
if not e.is_fatal:
raise # non-fatal → propagate immediately
if not context.remaining_validators:
raise # pool empty → propagate
# Replace leader with next validator
context.leader = context.remaining_validators.pop(0)
else:
# All replacement attempts exhausted
raise GenVMInternalError(
message="Leader idle: all replacements exhausted",
error_code=GenVMErrorCode.LLM_NO_PROVIDER,
causes=["ALL_LEADERS_IDLE"],
is_fatal=True,
is_leader=True,
)

context.transactions_processor.add_state_timestamp(
context.transaction.hash, "PROPOSING.TRANSACTION_EXECUTED"
Expand Down Expand Up @@ -2815,15 +2836,50 @@ def validator_timing_callback(step_name: str):
# Execute the transaction with a semaphore to limit the number of concurrent validators
sem = asyncio.Semaphore(8)

async def run_single_validator(validator: Node, index: int) -> Receipt:
# Build replacement pool: all validators minus those already assigned
assigned_addresses: set[str] = set()
if context.leader.get("address"):
assigned_addresses.add(context.leader["address"])
assigned_addresses.update(v["address"] for v in context.remaining_validators)
replacement_pool: list[dict] = [
n.validator.to_dict()
for n in context.validators_snapshot.nodes
if n.validator.to_dict()["address"] not in assigned_addresses
]
Comment on lines +2839 to +2848

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Select replacement validators via VRF instead of snapshot order.

The replacement pool is currently built in snapshot order, which bypasses the VRF-based rotation requirement for consensus.

🔁 VRF-based replacement selection
-        replacement_pool: list[dict] = [
-            n.validator.to_dict()
-            for n in context.validators_snapshot.nodes
-            if n.validator.to_dict()["address"] not in assigned_addresses
-        ]
+        not_used_validators: list[dict] = []
+        for node in context.validators_snapshot.nodes:
+            validator_dict = node.validator.to_dict()
+            if validator_dict["address"] not in assigned_addresses:
+                not_used_validators.append(validator_dict)
+
+        replacement_pool = get_validators_for_transaction(
+            not_used_validators, len(not_used_validators)
+        )

As per coding guidelines, "backend/consensus/**/*.py: Implement validator rotation using VRF (Verifiable Random Function) in the consensus system".

🤖 Prompt for AI Agents
In `@backend/consensus/base.py` around lines 2839 - 2848, The replacement_pool is
currently built in snapshot order which bypasses VRF-based rotation; change the
logic that builds replacement_pool so it first filters out assigned_addresses
(using context.leader["address"] and context.remaining_validators) and then
computes a VRF score for each remaining validator from
context.validators_snapshot.nodes (use the project's VRF utility, e.g.,
compute_vrf_score or vrf_sign/verify API) and sort/select validators by that VRF
output to form replacement_pool instead of relying on snapshot order; ensure you
keep the same validator-to-dict conversion (validator.to_dict()) after selecting
by VRF.

pool_lock = asyncio.Lock()

async def pop_replacement() -> dict | None:
async with pool_lock:
return replacement_pool.pop(0) if replacement_pool else None

def _is_fatal_error(receipt: Receipt) -> bool:
raw_error = (receipt.genvm_result or {}).get("raw_error")
return isinstance(raw_error, dict) and raw_error.get("fatal") is True

async def run_single_validator(validator_dict: dict, index: int) -> Receipt:
async with sem:
context.transactions_processor.add_state_timestamp(
context.transaction.hash, f"COMMITTING.VALIDATOR_{index}_START"
)
result = await validator.exec_transaction(context.transaction)
context.transactions_processor.add_state_timestamp(
context.transaction.hash, f"COMMITTING.VALIDATOR_{index}_END"
)
current = validator_dict
for attempt in range(MAX_IDLE_REPLACEMENTS + 1):
node = create_validator_node(context, current, index)
context.transactions_processor.add_state_timestamp(
context.transaction.hash,
f"COMMITTING.VALIDATOR_{index}_START" f".attempt_{attempt}",
)
result = await node.exec_transaction(context.transaction)
context.transactions_processor.add_state_timestamp(
context.transaction.hash,
f"COMMITTING.VALIDATOR_{index}_END" f".attempt_{attempt}",
)

if _is_fatal_error(result) and attempt < MAX_IDLE_REPLACEMENTS:
replacement = await pop_replacement()
if replacement is not None:
current = replacement
continue
break

if _is_fatal_error(result):
result.vote = Vote.IDLE
return result
Comment on lines +2855 to 2883

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Check Vote enum definition for IDLE member
echo "=== Searching for Vote enum definition ==="
rg -n "class Vote" --type=py -A 20

echo -e "\n=== Checking for IDLE in Vote enum ==="
rg -n "IDLE" --type=py -B 2 -A 2 | head -50

Repository: genlayerlabs/genlayer-studio

Length of output: 4633


🏁 Script executed:

#!/bin/bash
# Check GenVMInternalError and is_fatal in leader path
echo "=== GenVMInternalError exception definition ==="
rg -n "class GenVMInternalError" --type=py -A 10

echo -e "\n=== is_fatal attribute usage ==="
rg -n "is_fatal" --type=py -B 3 -A 3

Repository: genlayerlabs/genlayer-studio

Length of output: 11310


🏁 Script executed:

#!/bin/bash
# Check raw_error and fatal in Node.exec_transaction
echo "=== Searching for raw_error and fatal in node base ==="
rg -n "raw_error" --type=py -B 2 -A 2

echo -e "\n=== genvm_result structure ==="
rg -n "genvm_result" --type=py -B 1 -A 1 | head -50

Repository: genlayerlabs/genlayer-studio

Length of output: 10474


🏁 Script executed:

#!/bin/bash
# Verify the exact structure created at line 861
echo "=== backend/node/base.py around line 861 ==="
sed -n '850,870p' backend/node/base.py

echo -e "\n=== backend/node/base.py around line 917 ==="
sed -n '910,930p' backend/node/base.py

Repository: genlayerlabs/genlayer-studio

Length of output: 1989


🏁 Script executed:

#!/bin/bash
# Double-check what raw_error key name is used in tests for validation
echo "=== Test expectations for raw_error key ==="
rg -n 'raw_error.*fatal' --type=py -B 1 -A 1 | head -40

Repository: genlayerlabs/genlayer-studio

Length of output: 2716


Fix inconsistent raw_error key naming in GenVMInternalError handling.

Vote.IDLE is correctly defined (backend/node/types.py:100), but the validator-path error detection has a critical inconsistency:

When GenVMInternalError is caught during validator execution (backend/node/base.py:861), the receipt's raw_error is created with "is_fatal" key:

"raw_error": {"causes": e.causes, "is_fatal": e.is_fatal}

However, both the leader path check (backend/node/base.py:917) and consensus layer validator replacement check (backend/consensus/base.py:2857) expect the "fatal" key:

if raw_error.get("fatal") is True:

This causes the _is_fatal_error function in the consensus layer to always return False for validators encountering GenVMInternalError, breaking the validator replacement mechanism. Change line 861 to use "fatal" instead of "is_fatal" to align with the detection logic and test expectations.

🤖 Prompt for AI Agents
In `@backend/consensus/base.py` around lines 2855 - 2883, The consensus validator
replacement check (_is_fatal_error in run_single_validator) expects raw_error to
include a "fatal" boolean, but GenVMInternalError currently sets "raw_error"
with "is_fatal"; locate the exception handler that builds the Receipt for
GenVMInternalError (the block that sets "raw_error": {"causes": e.causes,
"is_fatal": e.is_fatal}) and rename the key to "fatal" (i.e. set "raw_error":
{"causes": e.causes, "fatal": e.is_fatal}) so _is_fatal_error and leader-path
checks correctly detect fatal errors and trigger validator replacement.


# Leader evaluates validation function
Expand All @@ -2832,40 +2888,37 @@ async def run_single_validator(validator: Node, index: int) -> Receipt:
and len(context.consensus_data.leader_receipt) == 1
)

# Create validator node for the leader
# Build list of validator dicts to run
if validation_by_leader:
context.validator_nodes = [
create_validator_node(context, context.leader, 0)
]
validators_to_run = [context.leader] + context.remaining_validators
else:
context.validator_nodes = []
validators_to_run = list(context.remaining_validators)

context.transactions_processor.add_state_timestamp(
context.transaction.hash, "COMMITTING.LEADER_NODE_CREATED"
)
# Create validator nodes for each validator
start_index = 1 if validation_by_leader else 0
context.validator_nodes.extend(
[
create_validator_node(context, validator, start_index + index)
for index, validator in enumerate(context.remaining_validators)
]
)
context.transactions_processor.add_state_timestamp(
context.transaction.hash, "COMMITTING.VALIDATOR_NODES_CREATED"
context.transaction.hash, "COMMITTING.VALIDATORS_PREPARED"
)

# Execute the transaction on each validator node and gather the results
# Execute the transaction on each validator and gather the results
context.transactions_processor.add_state_timestamp(
context.transaction.hash, "COMMITTING.VALIDATORS_EXECUTION_START"
)

validation_tasks = [
run_single_validator(validator, index)
for index, validator in enumerate(context.validator_nodes)
run_single_validator(validator_dict, index)
for index, validator_dict in enumerate(validators_to_run)
]
context.validation_results = await asyncio.gather(*validation_tasks)

# If all validators voted IDLE, infrastructure is systemically broken
if all(r.vote == Vote.IDLE for r in context.validation_results):
raise GenVMInternalError(
message="All validators idle after replacements",
error_code=GenVMErrorCode.LLM_NO_PROVIDER,
causes=["ALL_VALIDATORS_IDLE"],
is_fatal=True,
is_leader=True,
)

context.transactions_processor.add_state_timestamp(
context.transaction.hash, "COMMITTING.VALIDATORS_EXECUTION_END"
)
Expand Down Expand Up @@ -2925,7 +2978,7 @@ async def handle(self, context):
# Process each validation result and update the context
for i, validation_result in enumerate(context.validation_results):
# Store the vote from each validator node
context.votes[context.validator_nodes[i].address] = (
context.votes[validation_result.node_config["address"]] = (
validation_result.vote.value
)

Expand All @@ -2934,14 +2987,21 @@ async def handle(self, context):
consensus_result = determine_consensus_from_votes(votes_list)

# Send event in rollup to communicate the votes are revealed
# Vote.IDLE maps to Vote.TIMEOUT for on-chain events (Solidity
# VoteType enum doesn't have IDLE)
for i, validation_result in enumerate(context.validation_results):
last_vote = i == len(context.validation_results) - 1
chain_vote = (
Vote.TIMEOUT
if validation_result.vote == Vote.IDLE
else validation_result.vote
)
context.consensus_service.emit_transaction_event(
"emitVoteRevealed",
validation_result.node_config,
context.transaction.hash,
validation_result.node_config["address"],
int(validation_result.vote),
int(chain_vote),
last_vote,
int(consensus_result) if last_vote else int(ConsensusResult.IDLE),
)
Expand Down
27 changes: 17 additions & 10 deletions backend/consensus/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,32 @@

def determine_consensus_from_votes(votes_list: list[str]) -> ConsensusResult:
"""
Determine consensus from a list of votes.
Determine consensus from a list of votes using actual majority (>50%).

IDLE votes count as DISAGREE (couldn't participate = didn't agree).

Args:
votes_list: List of vote strings

Returns:
ConsensusResult: The consensus result
"""
total = len(votes_list)
majority = total / 2 # need strictly more than half

agree_count = votes_list.count(Vote.AGREE.value)
disagree_count = votes_list.count(Vote.DISAGREE.value)
timeout_count = votes_list.count(Vote.TIMEOUT.value)
idle_count = votes_list.count(Vote.IDLE.value)

if timeout_count > agree_count and timeout_count > disagree_count:
consensus_result = ConsensusResult.TIMEOUT
elif agree_count > disagree_count and agree_count > timeout_count:
consensus_result = ConsensusResult.MAJORITY_AGREE
elif disagree_count > agree_count and disagree_count > timeout_count:
consensus_result = ConsensusResult.MAJORITY_DISAGREE
else:
consensus_result = ConsensusResult.NO_MAJORITY
# IDLE = couldn't participate → counts as disagree
effective_disagree = disagree_count + idle_count

return consensus_result
if agree_count > majority:
return ConsensusResult.MAJORITY_AGREE
elif effective_disagree > majority:
return ConsensusResult.MAJORITY_DISAGREE
elif timeout_count > majority:
return ConsensusResult.TIMEOUT
else:
return ConsensusResult.NO_MAJORITY
19 changes: 16 additions & 3 deletions backend/node/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def get_simulator_chain_id() -> int:

def _filter_genvm_log_by_level(genvm_log: list[dict]) -> list[dict]:
"""
Filter genvm_log entries based on configured LOG_LEVEL with a minimum of WARNING.
Filter genvm_log entries based on configured LOG_LEVEL.
Only includes log entries that meet or exceed the effective threshold.
"""
# Get configured log level from environment
Expand All @@ -80,8 +80,7 @@ def _filter_genvm_log_by_level(genvm_log: list[dict]) -> list[dict]:
}

# Get numeric threshold for configured level (default to INFO if unknown)
# Enforce minimum WARNING level regardless of configuration
threshold = max(level_map.get(configured_level, logging.INFO), logging.WARNING)
threshold = level_map.get(configured_level, logging.INFO)

# Filter log entries
filtered_logs = []
Expand Down Expand Up @@ -912,5 +911,19 @@ async def _run_genvm(
)

if self.validator_mode == ExecutionMode.LEADER:
# Fatal user errors (infrastructure failures) → raise for consensus-level
# replacement. The consensus layer will retry with a replacement leader.
raw_error = (result.genvm_result or {}).get("raw_error") or {}
if raw_error.get("fatal") is True:
raise genvmbase.GenVMInternalError(
message=(
f"Leader fatal error:"
f" {result.genvm_result.get('error_code')}"
),
error_code=result.genvm_result.get("error_code"),
causes=raw_error.get("causes", []),
is_fatal=True,
is_leader=True,
)
return result
return self._set_vote(result)
13 changes: 13 additions & 0 deletions backend/node/genvm/error_codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
__all__ = (
"GenVMErrorCode",
"GenVMInternalError",
"LLM_ERROR_CODES",
"LUA_CAUSE_TO_CODE",
"RATE_LIMIT_STATUSES",
"extract_error_code",
Expand Down Expand Up @@ -90,6 +91,18 @@ class GenVMErrorCode(StrEnum):
# HTTP status codes that indicate rate limiting
RATE_LIMIT_STATUSES: set[int] = {429, 503, 529}

# LLM error codes that indicate infrastructure/provider failures (not contract bugs).
# These should trigger Vote.TIMEOUT for validators and GenVMInternalError for leaders.
LLM_ERROR_CODES: frozenset[str] = frozenset(
{
GenVMErrorCode.LLM_RATE_LIMITED,
GenVMErrorCode.LLM_NO_PROVIDER,
GenVMErrorCode.LLM_PROVIDER_ERROR,
GenVMErrorCode.LLM_INVALID_API_KEY,
GenVMErrorCode.LLM_TIMEOUT,
}
)

# Regex pattern for extracting fatal flag from ModuleError
_FATAL_PATTERN = re.compile(r"fatal:\s*(true|false)")

Expand Down
2 changes: 2 additions & 0 deletions backend/node/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ class Vote(Enum):
DISAGREE = "disagree"
TIMEOUT = "timeout"
DETERMINISTIC_VIOLATION = "deterministic_violation"
IDLE = "idle"

@classmethod
def from_string(cls, value: str) -> "Vote":
Expand All @@ -112,6 +113,7 @@ def __int__(self) -> int:
Vote.DISAGREE: 2,
Vote.TIMEOUT: 3,
Vote.DETERMINISTIC_VIOLATION: 4,
Vote.IDLE: 5,
}
return values[self]

Expand Down
11 changes: 11 additions & 0 deletions backend/protocol_rpc/app_lifespan.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
from backend.protocol_rpc.health import (
start_background_health_checker,
stop_background_health_checker,
record_genvm_execution_success,
record_genvm_execution_failure,
)
from backend.services.usage_metrics_service import UsageMetricsService
import backend.validators as validators
Expand Down Expand Up @@ -244,6 +246,15 @@ async def rpc_app_lifespan(app, settings: RPCAppSettings) -> AsyncIterator[RPCAp

genvm_manager = await create_genvm_manager()

# Wire GenVM execution failure tracking callbacks (same pattern as worker_service.py)
from backend.node.genvm.origin.base_host import set_genvm_callbacks

set_genvm_callbacks(
on_success=record_genvm_execution_success,
on_failure=record_genvm_execution_failure,
)
logger.info("[STARTUP] GenVM execution failure tracking callbacks registered")

validators_session = db_manager.open_session()
# Manages the validators registry with locking for concurrent access
# Handles LLM and web modules and configuration file changes
Expand Down
Loading
Loading