Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 93 additions & 11 deletions backend/consensus/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ class ConsensusWorker:
This reuses the exec_transaction logic from ConsensusAlgorithm.
"""

MAX_GENERIC_ERROR_RETRIES = 3

def __init__(
self,
get_session: Callable[[], Session],
Expand Down Expand Up @@ -123,6 +125,13 @@ def __init__(
os.environ.get("NO_VALIDATORS_BASE_BACKOFF_SECONDS", "30")
)

# Track retry counts for transactions that failed due to generic errors
# Key: transaction_hash, Value: {"count": int, "last_attempt": float, "last_error": str}
self._generic_error_retries: dict[str, dict] = {}
self._generic_error_base_backoff = float(
os.environ.get("GENERIC_ERROR_BASE_BACKOFF_SECONDS", "10")
)

# Initialize usage metrics service for reporting transaction metrics
self.usage_metrics_service = UsageMetricsService()

Expand Down Expand Up @@ -618,7 +627,7 @@ async def _transaction_context(
logger.error(
f"[Worker {self.worker_id}] GenVM internal error during {tx_type} {tx_hash}: "
f"code={e.error_code}, causes={e.causes}, is_fatal={e.is_fatal}, "
f"is_leader={e.is_leader}, message={e}"
f"is_leader={e.is_leader}, message={e}, detail={e.detail}"
)
session.rollback()

Expand Down Expand Up @@ -658,6 +667,7 @@ async def _transaction_context(
f"[Worker {self.worker_id}] Error processing {tx_type} {tx_hash}: {e}"
)
session.rollback()
await self._handle_generic_error_retry(tx_hash, e)
finally:
# Clear current transaction tracking
self.current_transactions.pop(tx_hash, None)
Expand Down Expand Up @@ -837,6 +847,8 @@ async def process_transaction(self, transaction_data: dict, session: Session):
# Clean up retry tracking on success
if transaction.hash in self._no_validators_retries:
del self._no_validators_retries[transaction.hash]
if transaction.hash in self._generic_error_retries:
del self._generic_error_retries[transaction.hash]

except NoValidatorsAvailableError:
# Handle no-validators case with retry logic and backoff
Expand Down Expand Up @@ -955,6 +967,59 @@ async def _handle_no_validators_retry(
f"next attempt in {backoff}s"
)

async def _handle_generic_error_retry(self, tx_hash: str, error: Exception):
"""
Handle retry logic for generic errors during transaction processing.
Implements exponential backoff and cancels after MAX_GENERIC_ERROR_RETRIES.

Args:
tx_hash: Transaction hash
error: The exception that occurred
"""
retry_info = self._generic_error_retries.get(
tx_hash, {"count": 0, "last_attempt": 0, "last_error": ""}
)
retry_info["count"] += 1
retry_info["last_attempt"] = time.time()
retry_info["last_error"] = str(error)
self._generic_error_retries[tx_hash] = retry_info

if retry_info["count"] >= self.MAX_GENERIC_ERROR_RETRIES:
# Cancel the transaction after max retries
logger.error(
f"[Worker {self.worker_id}] Transaction {tx_hash} canceled after "
f"{retry_info['count']} generic error retries - last error: {error}"
)
with self.get_session() as cancel_session:
tx = cancel_session.query(Transactions).filter_by(hash=tx_hash).one()
tx.status = TransactionStatus.CANCELED
tx.consensus_data = {
"error": "max_generic_retries_exceeded",
"last_error": str(error),
"retries": retry_info["count"],
}
cancel_session.commit()

# Send WebSocket notification
await ConsensusAlgorithm.dispatch_transaction_status_update(
TransactionsProcessor(cancel_session),
tx_hash,
TransactionStatus.CANCELED,
self.msg_handler,
)

# Clean up retry tracking
del self._generic_error_retries[tx_hash]
else:
backoff = self._generic_error_base_backoff * (
2 ** (retry_info["count"] - 1)
)
logger.warning(
f"[Worker {self.worker_id}] Generic error for {tx_hash}, "
f"retry {retry_info['count']}/{self.MAX_GENERIC_ERROR_RETRIES}, "
f"next attempt in {backoff}s - error: {error}"
)

async def _process_upgrade_transaction(
self, transaction_data: dict, session: Session
):
Expand Down Expand Up @@ -1293,7 +1358,7 @@ def _log_query_result(

def _is_in_backoff(self, transaction_data: dict) -> bool:
"""
Check if a transaction is in backoff due to no validators.
Check if a transaction is in backoff due to no validators or generic errors.

Args:
transaction_data: Transaction data dictionary
Expand All @@ -1302,18 +1367,35 @@ def _is_in_backoff(self, transaction_data: dict) -> bool:
True if transaction is in backoff period, False otherwise
"""
tx_hash = transaction_data["hash"]

# Check no-validators backoff
retry_info = self._no_validators_retries.get(tx_hash)
if not retry_info:
return False
if retry_info:
backoff = self._no_validators_base_backoff * (
2 ** (retry_info["count"] - 1)
)
time_since_last = time.time() - retry_info["last_attempt"]
if time_since_last < backoff:
logger.debug(
f"[Worker {self.worker_id}] Transaction {tx_hash} in no-validators backoff "
f"({time_since_last:.1f}s < {backoff}s)"
)
return True

backoff = self._no_validators_base_backoff * (2 ** (retry_info["count"] - 1))
time_since_last = time.time() - retry_info["last_attempt"]
if time_since_last < backoff:
logger.debug(
f"[Worker {self.worker_id}] Transaction {tx_hash} in backoff "
f"({time_since_last:.1f}s < {backoff}s)"
# Check generic error backoff
generic_info = self._generic_error_retries.get(tx_hash)
if generic_info:
backoff = self._generic_error_base_backoff * (
2 ** (generic_info["count"] - 1)
)
return True
time_since_last = time.time() - generic_info["last_attempt"]
if time_since_last < backoff:
logger.debug(
f"[Worker {self.worker_id}] Transaction {tx_hash} in generic-error backoff "
f"({time_since_last:.1f}s < {backoff}s)"
)
return True

return False

async def _process_transaction_task(self, transaction_data: dict):
Expand Down
5 changes: 5 additions & 0 deletions backend/consensus/worker_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,11 @@ def health_check():
"active_task_count": len(worker._active_tasks),
"max_parallel_txs": worker.max_parallel_txs,
"restart_count": worker_restart_count,
"generic_error_retries": (
len(worker._generic_error_retries)
if hasattr(worker, "_generic_error_retries")
else 0
),
**metrics,
}

Expand Down
1 change: 1 addition & 0 deletions backend/node/genvm/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ def provide_result(
error_code=error_code,
causes=causes,
is_fatal=is_fatal,
detail=error_str[:1000],
)
else:
raise Exception(f"invalid result {res.result_kind}")
Expand Down
2 changes: 2 additions & 0 deletions backend/node/genvm/error_codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,14 @@ def __init__(
causes: list[str],
is_fatal: bool,
is_leader: bool | None = None,
detail: str | None = None,
):
super().__init__(message)
self.error_code = error_code
self.causes = causes
self.is_fatal = is_fatal
self.is_leader = is_leader
self.detail = detail

def __repr__(self) -> str:
return (
Expand Down
123 changes: 65 additions & 58 deletions backend/protocol_rpc/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@
from backend.node.base import Manager as GenVMManager
import asyncio

# Limit concurrent GenVM executions on the jsonrpc path to prevent uvloop fd conflicts.
# Workers use asyncio.Semaphore(8) in consensus/base.py; gen_call had none, allowing
# unbounded concurrent GenVM socket operations that cause fd registry collisions.
_GENVM_CONCURRENCY = int(os.environ.get("GENVM_MAX_CONCURRENT", "8"))
_genvm_semaphore = asyncio.Semaphore(_GENVM_CONCURRENCY)


####### ADMIN ACCESS CONTROL #######
def require_admin_access(func):
Expand Down Expand Up @@ -959,64 +965,65 @@ async def _gen_call_with_validator(
sim_config is not None and sim_config.genvm_datetime is not None
)

if type == "read":
# Pre-parse timestamp override and map errors
txn_dt = None
if sim_config and override_transaction_datetime:
try:
txn_dt = sim_config.genvm_datetime_as_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_method_call_data(data)
receipt = await node.get_contract_data(
from_address=from_address,
calldata=decoded_data.calldata,
state_status=state_status,
transaction_datetime=txn_dt,
)
elif type == "write":
txn_created_at = None
if sim_config and override_transaction_datetime:
try:
_ = sim_config.genvm_datetime_as_datetime # validation only
txn_created_at = sim_config.genvm_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_method_send_data(data)
receipt = await node.run_contract(
from_address=from_address,
calldata=decoded_data.calldata,
transaction_created_at=txn_created_at,
)
elif type == "deploy":
txn_created_at = None
if sim_config and override_transaction_datetime:
try:
_ = sim_config.genvm_datetime_as_datetime # validation only
txn_created_at = sim_config.genvm_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_deployment_data(data)
receipt = await node.deploy_contract(
from_address=from_address,
code_to_deploy=decoded_data.contract_code,
calldata=decoded_data.calldata,
transaction_created_at=txn_created_at,
)
else:
raise JSONRPCError(f"Invalid type: {type}")
async with _genvm_semaphore:
if type == "read":
# Pre-parse timestamp override and map errors
txn_dt = None
if sim_config and override_transaction_datetime:
try:
txn_dt = sim_config.genvm_datetime_as_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_method_call_data(data)
receipt = await node.get_contract_data(
from_address=from_address,
calldata=decoded_data.calldata,
state_status=state_status,
transaction_datetime=txn_dt,
)
elif type == "write":
txn_created_at = None
if sim_config and override_transaction_datetime:
try:
_ = sim_config.genvm_datetime_as_datetime # validation only
txn_created_at = sim_config.genvm_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_method_send_data(data)
receipt = await node.run_contract(
from_address=from_address,
calldata=decoded_data.calldata,
transaction_created_at=txn_created_at,
)
elif type == "deploy":
txn_created_at = None
if sim_config and override_transaction_datetime:
try:
_ = sim_config.genvm_datetime_as_datetime # validation only
txn_created_at = sim_config.genvm_datetime
except ValueError as e:
raise JSONRPCError(
code=-32602,
message=f"Invalid sim_config.genvm_datetime: {sim_config.genvm_datetime}",
data={},
) from e
decoded_data = transactions_parser.decode_deployment_data(data)
receipt = await node.deploy_contract(
from_address=from_address,
code_to_deploy=decoded_data.contract_code,
calldata=decoded_data.calldata,
transaction_created_at=txn_created_at,
)
else:
raise JSONRPCError(f"Invalid type: {type}")

# Return the result of the write method
if receipt.execution_result != ExecutionResultStatus.SUCCESS:
Expand Down
Loading
Loading