From b17343a2a1f73526254dc871a88968f631f2f33e Mon Sep 17 00:00:00 2001 From: Isaac T Date: Sat, 28 Mar 2026 15:40:14 -0400 Subject: [PATCH 1/4] chore: change Pebble path mode to LevelDB hash - cause in local Pebble take too much of my storage space --- bsc_cluster.sh | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/bsc_cluster.sh b/bsc_cluster.sh index 310e80a3..0292c747 100644 --- a/bsc_cluster.sh +++ b/bsc_cluster.sh @@ -16,7 +16,7 @@ gcmode="full" sleepBeforeStart=15 sleepAfterStart=10 -# stop geth client +# 1. Stop any previously running geth instances function exit_previous() { ValIdx=$1 # if no geth exist just skip it @@ -24,6 +24,7 @@ function exit_previous() { sleep ${sleepBeforeStart} } +# 2. Cleanup .local and copy initial keys for validators function create_validator() { rm -rf ${workspace}/.local mkdir -p ${workspace}/.local @@ -34,6 +35,7 @@ function create_validator() { done } +# 3. Build bsc geth client from source if configured function prepare_bsc_client() { if [ ${useLatestBscClient} = true ]; then if [ ! -f "${workspace}/bsc/Makefile" ]; then @@ -43,7 +45,9 @@ function prepare_bsc_client() { cd ${workspace}/bsc && git pull && make geth && mv -f ${workspace}/bsc/build/bin/geth ${workspace}/bin/ fi } -# reset genesis, but keep edited genesis-template.json + +# 4. Reset genesis submodule and install dependencies (Poetry, NPM, Forge) +# This prepares the environment for generating the genesis block function reset_genesis() { if [ ! -f "${workspace}/genesis/genesis-template.json" ]; then cd ${workspace} && git submodule update --init --recursive genesis @@ -67,6 +71,7 @@ function reset_genesis() { git clone https://github.com/dapphub/ds-test } +# 5. Generate validator configurations, hardfork times, and the final genesis.json function prepare_config() { rm -f ${workspace}/genesis/validators.conf @@ -125,6 +130,7 @@ function prepare_config() { cp genesis-dev.json genesis.json } +# 6. Initialize the geth network for each node using the generated genesis.json function initNetwork() { cd ${workspace} for ((i = 0; i < size; i++)); do @@ -168,17 +174,13 @@ function initNetwork() { sed -i -e '/""/d' ${workspace}/.local/node${i}/config.toml # init genesis initLog=${workspace}/.local/node${i}/init.log - if [ $i -eq 0 ] ; then - ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 - else - ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 - fi + ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/node${i}/*bsc.log* if [ ${EnableSentryNode} = true ]; then sed -i -e '/""/d' ${workspace}/.local/sentry${i}/config.toml initLog=${workspace}/.local/sentry${i}/init.log - ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/sentry${i}/*bsc.log* fi done @@ -186,7 +188,7 @@ function initNetwork() { sed -i -e '/""/d' ${workspace}/.local/fullnode0/config.toml sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml initLog=${workspace}/.local/fullnode0/init.log - ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/fullnode0/*bsc.log* fi } @@ -229,6 +231,7 @@ function start_node() { >> ${datadir}/bsc-node.log 2>&1 & } +# 7. Start the nodes (Validators, Sentry, Full) in the background function native_start() { PassedForkTime=`cat ${workspace}/.local/node0/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'` LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY}) @@ -275,6 +278,7 @@ function native_start() { sleep ${sleepAfterStart} } +# 8. Use create-validator tool to register validator nodes on StakeHub function register_stakehub(){ # wait feynman enable sleep 45 @@ -284,18 +288,20 @@ function register_stakehub(){ done } +# Command dispatcher CMD=$1 ValidatorIdx=$2 case ${CMD} in reset) - exit_previous - create_validator - prepare_bsc_client - reset_genesis - prepare_config - initNetwork - native_start - register_stakehub + # The 'reset' flow perform a clean initialization of the entire local cluster: + exit_previous # Step 1: Kill old processes + create_validator # Step 2: Prepare keys and .local/ + prepare_bsc_client # Step 3: Build geth if necessary + reset_genesis # Step 4: Setup genesis deps (Forge, Poetry, etc) + prepare_config # Step 5: Generate genesis.json and node configs + initNetwork # Step 6: Initialize Geth data directories + native_start # Step 7: Start the cluster nodes + register_stakehub # Step 8: Register validators with the network ;; stop) exit_previous $ValidatorIdx From 30842b939949a42b8f50ec7381d87365620f4e6f Mon Sep 17 00:00:00 2001 From: Isaac T Date: Mon, 30 Mar 2026 13:58:10 -0400 Subject: [PATCH 2/4] feat: Update metrics and pprof bind addresses to 0.0.0.0 - allow monitoring access from the host machine via Docker port forwarding. --- bsc_cluster.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 bsc_cluster.sh diff --git a/bsc_cluster.sh b/bsc_cluster.sh old mode 100644 new mode 100755 index 0292c747..514613a1 --- a/bsc_cluster.sh +++ b/bsc_cluster.sh @@ -212,8 +212,8 @@ function start_node() { --rpc.allow-unprotected-txs --allow-insecure-unlock \ --ws --ws.addr 0.0.0.0 --ws.port ${ws_port} \ --http --http.addr 0.0.0.0 --http.port ${http_port} --http.corsdomain "*" \ - --metrics --metrics.addr localhost --metrics.port ${metrics_port} \ - --pprof --pprof.addr localhost --pprof.port ${pprof_port} \ + --metrics --metrics.addr 0.0.0.0 --metrics.port ${metrics_port} \ + --pprof --pprof.addr 0.0.0.0 --pprof.port ${pprof_port} \ --gcmode ${gcmode} --syncmode full --monitor.maliciousvote \ --rialtohash ${rialtoHash} \ --override.passedforktime ${PassedForkTime} \ From db69ba3c509fee8cd8a2e2cd3dc10b96cc3a7475 Mon Sep 17 00:00:00 2001 From: Isaac T Date: Mon, 30 Mar 2026 14:04:20 -0400 Subject: [PATCH 3/4] feat: forward pprof port --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a806e8f7..51604ba2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,10 @@ services: - "6062:6062" - "6064:6064" - "6066:6066" + - "7060:7060" + - "7062:7062" + - "7064:7064" + - "7066:7066" stdin_open: true tty: true entrypoint: /usr/local/bin/entrypoint.sh From 87c4ce8a7b876bcb6b29eca1fbde47eb98925099 Mon Sep 17 00:00:00 2001 From: Isaac T Date: Mon, 30 Mar 2026 14:04:40 -0400 Subject: [PATCH 4/4] docs: update readme --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b5463120..3cd5a07e 100644 --- a/README.md +++ b/README.md @@ -128,4 +128,29 @@ Here are some useful commands to monitor the cluster: - **Check Node Info**: `geth --exec "admin.nodeInfo" attach .local/node0/geth.ipc` - **Tail Logs**: - `tail -f .local/node0/bsc-node.log` \ No newline at end of file + `tail -f .local/node0/bsc-node.log` + +### Monitoring and Metrics + +The cluster is configured to expose internal metrics for monitoring. These are accessible from your **(Host)**: + +- **Prometheus Metrics**: `http://localhost:6060/debug/metrics/prometheus` +- **JSON Metrics**: `http://localhost:6060/debug/metrics` +- **Performance Profiling (pprof)**: `http://localhost:7060/debug/pprof/` + +**Port Mapping for Nodes:** + +| Node | RPC (HTTP/WS) | Metrics (Prometheus) | pprof (Debug) | +| :--- | :--- | :--- | :--- | +| **Node 0** | 8545 | 6060 | 7060 | +| **Node 1** | 8547 | 6062 | 7062 | +| **Node 2** | 8549 | 6064 | 7064 | +| **Node 3** | 8551 | 6066 | 7066 | + +### Storage Optimization + +To prevent rapid disk space exhaustion during local testing, this setup uses the following optimizations: + +- **DB Engine**: Forced to `leveldb` (more space-efficient than Pebble for small clusters). +- **State Scheme**: Set to `hash` to significantly reduce state storage size compared to the default path-based scheme. +- **Auto-Reset**: The `docker compose up` command triggers a full reset by default, ensuring you always start with a clean state. \ No newline at end of file