diff --git a/cli/src/main.rs b/cli/src/main.rs index 6094447..9d4ca1f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2,7 +2,13 @@ mod args; mod catalog; mod suite; -use std::{num::NonZeroUsize, process::ExitCode, rc::Rc}; +use std::{ + num::NonZeroUsize, + process::ExitCode, + rc::Rc, + sync::{Arc, OnceLock}, + time::Instant, +}; use anyhow::Context; use args::{Benchmarks, Checks, Cli, Command}; @@ -60,15 +66,18 @@ fn run_seed(seed: Option) -> u64 { } async fn interruptible( - run: impl AsyncFnOnce(watch::Receiver) -> Result<()>, + run: impl AsyncFnOnce(watch::Receiver, Arc>) -> Result<()>, ) -> Result { let (stop, receiver) = watch::channel(false); + let stop_requested_at = Arc::new(OnceLock::new()); + let interrupted_at = stop_requested_at.clone(); let interrupt = tokio::spawn(async move { if tokio::signal::ctrl_c().await.is_ok() { + interrupted_at.get_or_init(Instant::now); stop.send_replace(true); } }); - let outcome = run(receiver.clone()).await; + let outcome = run(receiver.clone(), stop_requested_at).await; interrupt.abort(); if *receiver.borrow() { if let Err(error) = outcome { @@ -129,10 +138,11 @@ async fn dispatch(command: Command) -> Result { } else { None }; - return interruptible(async move |stop| { + return interruptible(async move |stop, stop_requested_at| { let config = ExecutionConfig { duration: Some(args.duration), stop: Some(stop), + stop_requested_at: Some(stop_requested_at), environment, profile: args.profile, loop_mode: args.loop_mode, @@ -154,11 +164,12 @@ async fn dispatch(command: Command) -> Result { }, Command::Continuous(args) => { let seed = run_seed(args.seed); - return interruptible(async move |stop| { + return interruptible(async move |stop, stop_requested_at| { let config = ExecutionConfig { seed, duration: Some(args.duration), stop: Some(stop), + stop_requested_at: Some(stop_requested_at), profile: args.profile, continuous: true, ..ExecutionConfig::default() diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index d424f0d..1bbabae 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -17,7 +17,8 @@ use solana_instruction_error::InstructionError; use solana_rpc_client_types::{ config::{ RpcBlockConfig, RpcProgramAccountsConfig, RpcSendTransactionConfig, - RpcSignaturesForAddressConfig, RpcSimulateTransactionConfig, RpcTransactionConfig, + RpcSignatureStatusConfig, RpcSignaturesForAddressConfig, RpcSimulateTransactionConfig, + RpcTransactionConfig, }, request::RpcRequest, response::{ @@ -332,9 +333,32 @@ impl Api { pub async fn get_signature_statuses( &self, signatures: &[Signature], + ) -> Result>> { + self.signature_statuses(signatures, None).await + } + + pub async fn get_signature_statuses_with_history( + &self, + signatures: &[Signature], + ) -> Result>> { + self.signature_statuses( + signatures, + Some(RpcSignatureStatusConfig { search_transaction_history: true }), + ) + .await + } + + async fn signature_statuses( + &self, + signatures: &[Signature], + config: Option, ) -> Result>> { let signatures: Vec<_> = signatures.iter().map(ToString::to_string).collect(); - self.call_value(RpcRequest::GetSignatureStatuses, &(signatures,)).await + let method = RpcRequest::GetSignatureStatuses; + match config { + Some(config) => self.call_value(method, &(signatures, config)).await, + None => self.call_value(method, &(signatures,)).await, + } } /// Reads receipts without JSON's per-instruction base58 encoding, which is diff --git a/core/src/config.rs b/core/src/config.rs index 0a5bac0..0cc00ae 100644 --- a/core/src/config.rs +++ b/core/src/config.rs @@ -1,5 +1,10 @@ use clap::ValueEnum; -use std::{path::PathBuf, rc::Rc, time::Duration}; +use std::{ + path::PathBuf, + rc::Rc, + sync::{Arc, OnceLock}, + time::{Duration, Instant}, +}; use url::Url; /// An existing devnet-backed environment whose processes are not owned by RedSuite. @@ -57,6 +62,7 @@ pub struct ExecutionConfig { pub seed: u64, pub duration: Option, pub stop: Option>, + pub stop_requested_at: Option>>, pub environment: Option>, pub continuous: bool, } diff --git a/core/src/report/mod.rs b/core/src/report/mod.rs index 4a15358..b0fd344 100644 --- a/core/src/report/mod.rs +++ b/core/src/report/mod.rs @@ -197,7 +197,7 @@ pub fn run_id() -> &'static str { RUN_ID.get_or_init(|| format!("{}-{}", utc_stamp(), std::process::id())) } -fn campaign_dir() -> PathBuf { +pub fn campaign_dir() -> PathBuf { reports_dir().join(run_id()) } diff --git a/core/src/transport/events.rs b/core/src/transport/events.rs index 86bd18b..c472bdf 100644 --- a/core/src/transport/events.rs +++ b/core/src/transport/events.rs @@ -279,8 +279,7 @@ impl EventSubscriptions { topic, payload: PhantomData, }; - self.connection.subscribe(topic, params, (key, topic)).await?; - self.connection.await_ack((key, topic)).await?; + self.connection.subscribe_and_wait(topic, params, (key, topic)).await?; Ok(subscription) } diff --git a/core/src/transport/mod.rs b/core/src/transport/mod.rs index a7ce3a1..9ab4006 100644 --- a/core/src/transport/mod.rs +++ b/core/src/transport/mod.rs @@ -6,6 +6,7 @@ pub const NETWORK_TIMEOUT: Duration = Duration::from_secs(60); pub mod clock; pub mod events; pub mod http; +pub mod observations; pub mod rate; pub(crate) mod rpc; mod socket; diff --git a/core/src/transport/observations.rs b/core/src/transport/observations.rs new file mode 100644 index 0000000..17b8e41 --- /dev/null +++ b/core/src/transport/observations.rs @@ -0,0 +1,126 @@ +use std::{ + ops::ControlFlow::{Break, Continue}, + rc::Rc, +}; + +use pubkey::Pubkey; +use signature::Signature; +use tokio::sync::Notify; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; +use url::Url; + +use super::{ + socket::CloseReason, + subscription::{self, Connection, SubscriptionState, Topic}, +}; +use crate::Result; + +#[derive(Clone, Copy, PartialEq)] +enum Key { + Account(usize), + Signature(u64), +} + +#[derive(Clone, Copy)] +pub enum ObservationClose { + GoingAway, + Transport, + Other, +} + +#[derive(Default)] +struct Observer { + close_reason: Option, + closed: Rc, +} + +impl SubscriptionState for Observer { + type Key = Key; + + fn on_close(&mut self, reason: &CloseReason) { + self.close_reason = Some(match reason { + CloseReason::ServerClose(Some(frame)) if frame.code == CloseCode::Away => { + ObservationClose::GoingAway + } + CloseReason::Transport(_) => ObservationClose::Transport, + _ => ObservationClose::Other, + }); + self.closed.notify_waiters(); + } +} + +pub enum Notification { + Account { index: usize, data: Vec }, + Signature { id: u64, succeeded: bool }, +} + +pub struct ObservationSocket { + connection: Connection, +} + +impl ObservationSocket { + pub async fn connect( + url: &Url, + mut observe: impl FnMut(Notification) + 'static, + ) -> Result { + let connection = + Connection::connect(url, Observer::default(), move |_, key, payload| match key { + Key::Account(index) => { + if let Some(data) = subscription::account_data(payload) { + observe(Notification::Account { index, data }); + } + Continue(()) + } + Key::Signature(id) => { + let Some(result) = subscription::processed_signature(payload) else { + return Continue(()); + }; + observe(Notification::Signature { + id, + succeeded: result.err.is_none(), + }); + Break(()) + } + }) + .await?; + Ok(Self { connection }) + } + + pub async fn account(&self, index: usize, address: &Pubkey) -> Result<()> { + self.connection + .subscribe_and_wait( + Topic::Account, + &subscription::account_params(address), + Key::Account(index), + ) + .await + } + + pub async fn signature(&self, id: u64, signature: &Signature) -> Result<()> { + self.connection + .subscribe_and_wait( + Topic::Signature, + &subscription::signature_params(signature), + Key::Signature(id), + ) + .await + } + + pub async fn unsubscribe_signature(&self, id: u64) -> Result<()> { + self.connection.unsubscribe(Key::Signature(id), Topic::Signature).await + } + + pub fn is_open(&self) -> bool { + self.connection.check_error().is_ok() + } + + pub async fn closed(&self) -> ObservationClose { + let closed = self.connection.state().closed.clone(); + let notification = closed.notified(); + if let Some(reason) = self.connection.state().close_reason { + return reason; + } + notification.await; + self.connection.state().close_reason.expect("close notification has a reason") + } +} diff --git a/core/src/transport/subscription.rs b/core/src/transport/subscription.rs index ac81ff9..95d8dd1 100644 --- a/core/src/transport/subscription.rs +++ b/core/src/transport/subscription.rs @@ -20,7 +20,7 @@ use solana_rpc_client_types::{ RpcTransactionLogsConfig, RpcTransactionLogsFilter, }, request::RpcRequest, - response::Response, + response::{ProcessedSignatureResult, Response, RpcSignatureResult}, }; use tokio::{ sync::{Mutex, Notify}, @@ -221,6 +221,19 @@ impl Connection { Ok(()) } + pub(super) async fn subscribe_and_wait( + &self, + topic: Topic, + params: &impl Serialize, + key: S::Key, + ) -> Result<()> + where + S::Key: PartialEq, + { + self.subscribe(topic, params, key).await?; + self.await_ack(key).await + } + pub(super) fn state(&self) -> Ref<'_, S> { Ref::map(self.router.borrow(), |router| &router.state) } @@ -282,7 +295,7 @@ impl Connection { ) } - pub(super) async fn await_ack(&self, key: S::Key) -> Result<()> + async fn await_ack(&self, key: S::Key) -> Result<()> where S::Key: PartialEq, { @@ -339,6 +352,14 @@ impl Drop for Connection { } } +pub(super) fn processed_signature(payload: &LazyValue<'_>) -> Option { + let response: Response = json::from_str(payload.as_raw_str()).ok()?; + match response.value { + RpcSignatureResult::ProcessedSignature(result) => Some(result), + RpcSignatureResult::ReceivedSignature(_) => None, + } +} + pub(super) fn account_data(payload: &LazyValue<'_>) -> Option> { let payload: Response = json::from_str(payload.as_raw_str()).ok()?; payload.value.data.decode() diff --git a/core/src/transport/ws.rs b/core/src/transport/ws.rs index 9760fd2..74340f7 100644 --- a/core/src/transport/ws.rs +++ b/core/src/transport/ws.rs @@ -9,7 +9,6 @@ use anyhow::Context; use json::LazyValue; use pubkey::Pubkey; use signature::Signature; -use solana_rpc_client_types::response::{Response, RpcSignatureResult}; use tokio::{sync::oneshot, time}; use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use url::Url; @@ -224,11 +223,7 @@ impl SubscriptionState for SignatureState { impl SignatureState { fn observe(&mut self, id: u64, payload: &LazyValue<'_>, clock: &SendClock) -> ControlFlow<()> { - let Ok(Response { - value: RpcSignatureResult::ProcessedSignature(result), - .. - }) = json::from_str::>(payload.as_raw_str()) - else { + let Some(result) = subscription::processed_signature(payload) else { return Continue(()); }; if !self.pending.remove(&id) { @@ -273,13 +268,12 @@ impl SignatureConfirmations { self.connection.check_error()?; self.connection.state_mut().pending.insert(id); self.connection - .subscribe( + .subscribe_and_wait( Topic::Signature, &subscription::signature_params(signature), id, ) - .await?; - self.connection.await_ack(id).await + .await } pub async fn await_id(&self, id: u64) -> Result<()> { diff --git a/redline/src/scenarios/aperture/fixed_pool/mod.rs b/redline/src/scenarios/aperture/fixed_pool/mod.rs new file mode 100644 index 0000000..92a3832 --- /dev/null +++ b/redline/src/scenarios/aperture/fixed_pool/mod.rs @@ -0,0 +1,397 @@ +mod observations; +mod worker; + +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::{BufWriter, Write}, + rc::Rc, + time::{Duration, Instant}, +}; + +use anyhow::{anyhow, ensure, Context}; +use redsuite_core::{ + config::{LoopMode, ProfileValues}, + prep, + redline::Accounts, + report::{self, Unit}, + runner::spawn_workers, + BaseCtx, ChainCtx, ErCtx, Result, Scenario, ScenarioReport, +}; +use tokio::{ + sync::{mpsc, watch}, + time::{self, MissedTickBehavior}, +}; + +use self::observations::{Clock, Count, Sample}; +use crate::RunControl; + +const WRITABLE: usize = 16; +const SEED: u64 = 20_261_112; +const ACCOUNT_SPACE: u32 = 128; +const PAYER_LAMPORTS: u64 = 4_000_000_000; +const STARTUP_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone, Copy)] +pub(super) struct Profile { + pub(super) workers: usize, + pub(super) accounts: usize, + pub(super) payers: usize, + pub(super) sockets: usize, + pub(super) rate_per_worker: u32, + pub(super) concurrency: usize, + pub(super) http_concurrency: usize, + warmup: Duration, +} + +const PROFILES: ProfileValues = ProfileValues { + lite: Profile { + workers: 1, + accounts: 32, + payers: 16, + sockets: 4, + rate_per_worker: 1_000, + concurrency: 64, + http_concurrency: 32, + warmup: Duration::from_secs(5), + }, + full: Profile { + workers: 4, + accounts: 512, + payers: 256, + sockets: 16, + rate_per_worker: 24_000, + concurrency: 512, + http_concurrency: 128, + warmup: Duration::from_secs(60), + }, +}; + +pub struct FixedPool { + base: Rc, + er: ErCtx, + run: RunControl, + config: worker::Config, +} + +impl Scenario<(ScenarioReport, Result<()>)> for FixedPool { + async fn setup(base: Rc, er: ErCtx, _: ()) -> Result { + ensure!( + base.config().loop_mode == LoopMode::Open, + "fixed_pool requires --loop open" + ); + let run = RunControl::from_config(&base.config())?; + let profile = *PROFILES.select(base.config().profile); + let payers = base.funded_payers(profile.payers, PAYER_LAMPORTS).await?; + let accounts = Accounts::new(ACCOUNT_SPACE, er.identity()) + .init_batched(base.as_ref(), &payers, profile.accounts, true) + .await?; + er.await_clones(&accounts, ACCOUNT_SPACE as usize).await?; + let config = worker::Config { + profile, + rpc: er.api().url().clone(), + ws: er.ws_url().clone(), + accounts: accounts.into(), + payers: prep::payer_bytes(&payers), + stop: run.stop.clone(), + }; + Ok(Self { base, er, run, config }) + } + + async fn run(self) -> Result<(ScenarioReport, Result<()>)> { + let profile = self.config.profile; + let directory = report::campaign_dir(); + fs::create_dir_all(&directory)?; + let api_path = directory.join("fixed_pool-api.jsonl"); + let validator_path = directory.join("fixed_pool-validator.jsonl"); + let api = BufWriter::new( + File::create(&api_path).with_context(|| format!("creating {}", api_path.display()))?, + ); + let mut validator = BufWriter::new( + File::create(&validator_path) + .with_context(|| format!("creating {}", validator_path.display()))?, + ); + + let (ready, mut workers_ready) = mpsc::channel(profile.workers); + let (stop, worker_stop) = watch::channel(false); + let (start, clock) = watch::channel(None); + let workers = spawn_workers(profile.workers, move |worker| { + worker::run( + worker.index, + self.config.clone(), + ready.clone(), + clock.clone(), + worker_stop.clone(), + ) + }); + let mut requested_stop = self.run.stop.clone(); + let startup = time::timeout(STARTUP_TIMEOUT, async { + for _ in 0..profile.workers { + ensure!( + workers_ready.recv().await == Some(true), + "benchmark workers failed to initialize" + ); + } + Ok(()) + }); + let readiness = tokio::select! { + biased; + _ = requested_stop.wait_for(|stop| *stop) => Err(anyhow!("benchmark startup cancelled")), + result = startup => result.context("benchmark worker startup timed out").and_then(|result| result), + }; + if let Err(error) = readiness { + drop(start); + workers.join().await?; + return Err(error); + } + drop(workers_ready); + + let clock = Clock::new(profile.warmup, self.run.duration); + start.send_replace(Some(clock)); + let mut first_measured_sample = None; + let mut observed_stop_at = None; + let outcomes = { + let mut joined = std::pin::pin!(workers.join()); + let mut collecting = std::pin::pin!(collect_validator( + &self.er, + clock, + &mut validator, + &mut first_measured_sample, + )); + loop { + tokio::select! { + result = &mut joined => break result?, + _ = requested_stop.wait_for(|stop| *stop), if observed_stop_at.is_none() => { + observed_stop_at = Some(Instant::now()); + stop.send_replace(true); + } + error = &mut collecting => { + stop.send_replace(true); + joined.await?; + return Err(error); + } + } + } + }; + validator.flush()?; + + let mut by_second = Vec::::new(); + let mut worker_failures = Vec::new(); + for outcome in outcomes.into_iter().flatten() { + if let Err(error) = outcome.result { + worker_failures.push(format!("{error:#}")); + } + if by_second.len() < outcome.samples.len() { + by_second.resize_with(outcome.samples.len(), Sample::default); + } + for (total, sample) in by_second.iter_mut().zip(&outcome.samples) { + total.merge(sample); + } + } + let interrupted = *self.run.stop.borrow(); + let cutoff = self + .base + .config() + .stop_requested_at + .as_ref() + .and_then(|stop| stop.get().copied()) + .or(observed_stop_at) + .or_else(|| interrupted.then(Instant::now)) + .unwrap_or(clock.end) + .min(clock.end); + let measured_seconds = cutoff.saturating_duration_since(clock.begin).as_secs_f64(); + let total = write_api_samples(api, clock, measured_seconds, by_second)?; + + let metrics_captured = first_measured_sample.is_some_and(|at| at < cutoff); + let accounting = check_accounting(&total, &worker_failures, interrupted); + let completion = if accounting.is_ok() && !interrupted && !metrics_captured { + "incomplete" + } else { + self.run.completion(&accounting) + }; + let result = accounting.and_then(|()| { + ensure!( + interrupted || metrics_captured, + "benchmark captured no validator metrics during measurement" + ); + Ok(()) + }); + let mut report = ScenarioReport::ok() + .setting("profile", self.base.config().profile.name()) + .setting("completion", completion) + .setting("seed", SEED) + .setting("workers", profile.workers) + .setting("payers", profile.payers) + .setting("accounts", profile.accounts) + .setting("writable accounts per transaction", WRITABLE) + .setting("request mix", "80% writes / 20% getAccountInfo") + .setting( + "write submission policy", + "require signature subscription ACK and open socket", + ) + .setting("validator metrics captured", metrics_captured) + .setting( + "WebSockets per worker per subscription type", + profile.sockets, + ) + .setting( + "offered requests per second", + u64::from(profile.rate_per_worker) * profile.workers as u64, + ) + .setting("API time series", api_path.display()) + .setting("validator time series", validator_path.display()) + .metric("measured seconds", Unit::Seconds, measured_seconds); + for (name, value) in total.counters() { + report = report.metric(name, Unit::Count, value as f64); + } + for (name, _, mean, max) in total.timings() { + report = report.metric(format!("{name} mean us"), Unit::Micros, mean).metric( + format!("{name} max us"), + Unit::Micros, + max, + ); + } + if measured_seconds > 0.0 { + for (name, kind) in [ + ("signature-confirmed TPS", Count::Signature), + ("status-confirmed TPS", Count::Status), + ] { + report = + report.metric(name, Unit::Tps, total.count(kind) as f64 / measured_seconds); + } + } + Ok((report, result)) + } +} + +fn write_api_samples( + mut api: BufWriter, + clock: Clock, + measured_seconds: f64, + mut samples: Vec, +) -> Result { + let windows = (measured_seconds.ceil() as usize).max(samples.len()).max(1); + samples.resize_with(windows, Sample::default); + let mut total = Sample::default(); + for (second, sample) in samples.iter().enumerate() { + let counters: BTreeMap<_, _> = sample.counters().collect(); + let latencies: BTreeMap<_, _> = sample + .timings() + .map(|(name, count, mean, max)| { + ( + name, + json::json!({ "count": count, "mean_us": mean, "max_us": max }), + ) + }) + .collect(); + write_line( + &mut api, + &json::json!({ + "second": second, "unix_seconds": clock.unix + second as f64, + "phase": if second as f64 >= measured_seconds { "drain" } else { "measure" }, + "counters": counters, "latencies": latencies, + }), + )?; + total.merge(sample); + } + api.flush()?; + Ok(total) +} + +fn check_accounting(total: &Sample, worker_failures: &[String], interrupted: bool) -> Result<()> { + ensure!( + worker_failures.is_empty(), + "benchmark workers failed: {}", + worker_failures.join("; ") + ); + let count = |counter| total.count(counter); + let submitted = count(Count::Submitted); + ensure!( + interrupted || submitted > 0, + "benchmark submitted no transactions" + ); + ensure!( + submitted == count(Count::Accepted) + count(Count::SendError), + "HTTP requests did not reconcile" + ); + ensure!( + submitted + == count(Count::Signature) + + count(Count::SignatureError) + + count(Count::SignatureExpired), + "signature notifications did not reconcile" + ); + ensure!( + submitted == count(Count::Status) + count(Count::StatusError) + count(Count::StatusExpired), + "status observations did not reconcile" + ); + ensure!( + count(Count::AccountExpected) == count(Count::Account) + count(Count::AccountExpired), + "account notifications did not reconcile" + ); + Ok(()) +} + +async fn collect_validator( + er: &ErCtx, + clock: Clock, + out: &mut BufWriter, + first_measured_sample: &mut Option, +) -> anyhow::Error { + let mut samples = time::interval(Duration::from_secs(1)); + samples.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + samples.tick().await; + let now = Instant::now(); + let values = scrape(er).await; + if values.is_some() && clock.measured(now) { + first_measured_sample.get_or_insert(now); + } + let elapsed = if now < clock.begin { + -clock.begin.duration_since(now).as_secs_f64() + } else { + now.duration_since(clock.begin).as_secs_f64() + }; + let row = json::json!({ + "elapsed_seconds": elapsed, + "unix_seconds": clock.unix + elapsed, + "values": values, + }); + if let Err(error) = write_line(out, &row) { + return error.context("saving benchmark validator samples"); + } + } +} + +async fn scrape(er: &ErCtx) -> Option> { + let metrics = time::timeout(Duration::from_millis(500), er.scrape_metrics()) + .await + .ok()? + .ok()?; + let values: BTreeMap<_, _> = metrics + .0 + .into_iter() + .filter(|(name, value)| { + value.is_finite() + && (name.starts_with("engine_keeper_subscription_slow_consumer_disconnects") + || name.starts_with("engine_ledger_operation_duration_micros_count") + || name.starts_with("engine_ledger_operation_duration_micros_sum") + || name.starts_with("engine_keeper_operation_duration_micros_count") + || name.starts_with("engine_keeper_operation_duration_micros_sum") + || name.starts_with("engine_processor_blocked_transactions") + || matches!( + name.as_str(), + "engine_ledger_transactions" + | "engine_ledger_blocks" + | "engine_ledger_superblocks" + | "engine_ledger_pending_transactions" + )) + }) + .collect(); + (!values.is_empty()).then_some(values) +} + +fn write_line(out: &mut impl Write, value: &json::Value) -> Result<()> { + let line = json::to_string(value).context("serializing benchmark JSON")?; + writeln!(out, "{line}").context("writing benchmark JSON line") +} diff --git a/redline/src/scenarios/aperture/fixed_pool/observations.rs b/redline/src/scenarios/aperture/fixed_pool/observations.rs new file mode 100644 index 0000000..23404d0 --- /dev/null +++ b/redline/src/scenarios/aperture/fixed_pool/observations.rs @@ -0,0 +1,372 @@ +use std::{ + collections::HashMap, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use redsuite_core::{programs::CellHeader, transport::observations::Notification}; +use signature::Signature; + +pub(super) const EXPIRY: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy)] +pub(super) struct Clock { + pub(super) begin: Instant, + pub(super) end: Instant, + pub(super) unix: f64, +} + +impl Clock { + pub(super) fn new(warmup: Duration, measured: Duration) -> Self { + let begin = Instant::now() + warmup; + let unix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs_f64() + + warmup.as_secs_f64(); + Self { + begin, + end: begin + measured, + unix, + } + } + + pub(super) fn measured(self, at: Instant) -> bool { + self.begin <= at && at < self.end + } +} + +#[derive(Clone, Copy)] +pub(super) enum Count { + Submitted, + Accepted, + SendError, + Read, + ReadError, + Signature, + SignatureError, + SignatureExpired, + Status, + StatusError, + StatusExpired, + StatusRpcError, + AccountExpected, + Account, + AccountExpired, + AccountDuplicate, + AckError, + WriteSkipped, + Disconnect, + GoingAway, + TransportDisconnected, + Reconnect, + Length, +} + +const COUNTERS: [&str; Count::Length as usize] = [ + "transactions submitted", + "transactions accepted", + "transaction submission errors", + "account reads", + "account read errors", + "signature confirmations", + "signature execution errors", + "signature timeouts", + "status confirmations", + "status execution errors", + "status unobserved", + "status RPC errors", + "account updates expected", + "account updates received", + "account update timeouts", + "duplicate account updates", + "signature ACK errors", + "writes skipped by signature gate", + "WebSocket disconnects", + "WebSocket Going Away closes", + "WebSocket transport errors", + "WebSocket reconnects", +]; + +#[derive(Clone, Copy)] +pub(super) enum Latency { + Ack, + Send, + Read, + Signature, + Status, + Account, + Length, +} + +const LATENCIES: [&str; Latency::Length as usize] = [ + "signature subscription ACK", + "transaction response", + "account read response", + "signature notification", + "independent status observation", + "account notification", +]; + +#[derive(Default)] +struct Timing { + count: u64, + sum_us: u128, + max_us: u128, +} + +#[derive(Default)] +pub(super) struct Sample { + counts: [u64; Count::Length as usize], + times: [Timing; Latency::Length as usize], +} + +impl Sample { + pub(super) fn merge(&mut self, other: &Self) { + for (total, value) in self.counts.iter_mut().zip(&other.counts) { + *total += value; + } + for (total, value) in self.times.iter_mut().zip(&other.times) { + total.count += value.count; + total.sum_us += value.sum_us; + total.max_us = total.max_us.max(value.max_us); + } + } + + pub(super) fn counters(&self) -> impl Iterator + '_ { + COUNTERS.into_iter().zip(self.counts) + } + + pub(super) fn timings(&self) -> impl Iterator + '_ { + LATENCIES + .into_iter() + .zip(&self.times) + .filter(|(_, timing)| timing.count > 0) + .map(|(name, timing)| { + ( + name, + timing.count, + timing.sum_us as f64 / timing.count as f64, + timing.max_us as f64, + ) + }) + } + + pub(super) fn count(&self, counter: Count) -> u64 { + self.counts[counter as usize] + } +} + +#[derive(Default)] +pub(super) struct Observations { + pub(super) clock: Option, + pub(super) samples: Vec, +} + +impl Observations { + fn sample(&mut self, started: Instant, at: Instant) -> Option<&mut Sample> { + let clock = self.clock.filter(|clock| clock.measured(started))?; + let second = at.duration_since(clock.begin).as_secs() as usize; + if second >= self.samples.len() { + self.samples.resize_with(second + 1, Sample::default); + } + Some(&mut self.samples[second]) + } + + pub(super) fn count(&mut self, counter: Count, started: Instant, at: Instant, n: u64) { + if n != 0 { + if let Some(sample) = self.sample(started, at) { + sample.counts[counter as usize] += n; + } + } + } + + pub(super) fn record( + &mut self, + started: Instant, + at: Instant, + counter: Option, + latency: Option, + ) { + let Some(sample) = self.sample(started, at) else { return }; + if let Some(counter) = counter { + sample.counts[counter as usize] += 1; + } + if let Some(metric) = latency { + let us = at.duration_since(started).as_micros(); + let timing = &mut sample.times[metric as usize]; + timing.count += 1; + timing.sum_us += us; + timing.max_us = timing.max_us.max(us); + } + } +} + +struct Pending { + signature: Signature, + signature_pending: bool, + status_pending: bool, + sent: Instant, + http_done: Option, + expected: u128, + seen: u128, +} + +impl Pending { + fn notification_expired(&self, now: Instant) -> bool { + self.http_done.is_some_and(|done| now >= done + EXPIRY) + } + + fn status_expired(&self, now: Instant) -> bool { + now >= self.sent + EXPIRY + } +} + +#[derive(Default)] +pub(super) struct Tracker { + pending: HashMap, + pub(super) data: Observations, +} + +impl Tracker { + pub(super) fn submitted( + &mut self, + id: u64, + signature: Signature, + sent: Instant, + expected: u128, + ) { + self.pending.insert( + id, + Pending { + signature, + signature_pending: true, + status_pending: true, + sent, + http_done: None, + expected, + seen: 0, + }, + ); + if let Some(sample) = self.data.sample(sent, sent) { + sample.counts[Count::Submitted as usize] += 1; + sample.counts[Count::AccountExpected as usize] += u64::from(expected.count_ones()); + } + } + + pub(super) fn http_done(&mut self, id: u64, now: Instant, ok: bool) { + let tx = self + .pending + .get_mut(&id) + .expect("submitted transaction has a pending HTTP request"); + tx.http_done = Some(now); + self.data.record( + tx.sent, + now, + Some(if ok { Count::Accepted } else { Count::SendError }), + ok.then_some(Latency::Send), + ); + } + + pub(super) fn observe(&mut self, notification: Notification, now: Instant) { + match notification { + Notification::Account { index, data } => { + let Some(tx) = + CellHeader::written_id(&data).and_then(|id| self.pending.get_mut(&id)) + else { + return; + }; + let bit = 1u128 << index; + if tx.notification_expired(now) || tx.expected & bit == 0 { + return; + } + let duplicate = tx.seen & bit != 0; + tx.seen |= bit; + self.data.record( + tx.sent, + now, + Some(if duplicate { Count::AccountDuplicate } else { Count::Account }), + (!duplicate).then_some(Latency::Account), + ); + } + Notification::Signature { id, succeeded } => { + let Some(tx) = self.pending.get_mut(&id) else { return }; + if !tx.signature_pending || tx.notification_expired(now) { + return; + } + tx.signature_pending = false; + self.data.record( + tx.sent, + now, + Some(if succeeded { Count::Signature } else { Count::SignatureError }), + succeeded.then_some(Latency::Signature), + ); + } + } + } + + pub(super) fn status(&mut self, id: u64, succeeded: bool, now: Instant) { + let Some(tx) = self.pending.get_mut(&id) else { return }; + if !tx.status_pending || tx.status_expired(now) { + return; + } + tx.status_pending = false; + self.data.record( + tx.sent, + now, + Some(if succeeded { Count::Status } else { Count::StatusError }), + Some(Latency::Status), + ); + } + + pub(super) fn restore(&self, socket: usize, connections: usize) -> Vec<(u64, Signature)> { + self.pending + .iter() + .filter_map(|(&id, tx)| { + (id as usize % connections == socket && tx.signature_pending) + .then_some((id, tx.signature)) + }) + .collect() + } + + pub(super) fn signature_pending(&self, id: u64) -> bool { + self.pending.get(&id).is_some_and(|tx| tx.signature_pending) + } + + pub(super) fn statuses(&self, now: Instant) -> Vec<(u64, Signature)> { + self.pending + .iter() + .filter_map(|(&id, tx)| { + (tx.status_pending && !tx.status_expired(now)).then_some((id, tx.signature)) + }) + .collect() + } + + pub(super) fn reap(&mut self, now: Instant) -> Vec { + let mut unsubscribe = Vec::new(); + self.pending.retain(|&id, tx| { + if tx.http_done.is_none() { + return true; + } + if tx.status_pending && tx.status_expired(now) { + tx.status_pending = false; + self.data.count(Count::StatusExpired, tx.sent, now, 1); + } + if tx.notification_expired(now) { + if tx.signature_pending { + unsubscribe.push(id); + self.data.count(Count::SignatureExpired, tx.sent, now, 1); + } + self.data.count( + Count::AccountExpired, + tx.sent, + now, + u64::from((tx.expected & !tx.seen).count_ones()), + ); + return false; + } + tx.status_pending || tx.signature_pending || tx.expected != tx.seen + }); + unsubscribe + } + + pub(super) fn pending(&self) -> bool { + !self.pending.is_empty() + } +} diff --git a/redline/src/scenarios/aperture/fixed_pool/worker.rs b/redline/src/scenarios/aperture/fixed_pool/worker.rs new file mode 100644 index 0000000..9391ccf --- /dev/null +++ b/redline/src/scenarios/aperture/fixed_pool/worker.rs @@ -0,0 +1,438 @@ +use std::{ + cell::RefCell, + rc::Rc, + sync::Arc, + time::{Duration, Instant}, +}; + +use anyhow::{anyhow, Context}; +use futures_util::{ + future::{self, try_join_all}, + stream, StreamExt, TryStreamExt, +}; +use pubkey::Pubkey; +use redsuite_core::{ + prep, + programs::REDLINE, + random::{self, Rng}, + runner::{execute_for_raw, Pacing, SendTransaction}, + transport::observations::{ObservationClose, ObservationSocket}, + Api, Result, +}; +use signature::Signature; +use tokio::{ + sync::{mpsc, watch, Semaphore}, + task::JoinSet, + time::{self, MissedTickBehavior}, +}; +use url::Url; + +use super::{ + observations::{Clock, Count, Latency, Sample, Tracker, EXPIRY}, + Profile, SEED, WRITABLE, +}; + +const RPC_TIMEOUT: Duration = Duration::from_secs(5); +const POLL_INTERVAL: Duration = Duration::from_millis(100); +const STATUS_BATCH: usize = 256; +const STATUS_PARALLELISM: usize = 8; + +type SharedTracker = Rc>; +type SocketSlot = Rc>>>; + +#[derive(Clone)] +pub(super) struct Config { + pub(super) profile: Profile, + pub(super) rpc: Url, + pub(super) ws: Url, + pub(super) accounts: Arc>, + pub(super) payers: Arc>, + pub(super) stop: watch::Receiver, +} + +pub(super) struct Outcome { + pub(super) result: Result<()>, + pub(super) samples: Vec, +} + +pub(super) async fn run( + index: usize, + config: Config, + ready: mpsc::Sender, + mut start: watch::Receiver>, + worker_stop: watch::Receiver, +) -> Result> { + let profile = config.profile; + let pool_size = profile.accounts / profile.workers; + let accounts: Rc<[Pubkey]> = config.accounts[index * pool_size..(index + 1) * pool_size].into(); + let senders = prep::worker_senders( + &config.rpc, + &config.ws, + &config.payers, + index, + profile.workers, + ); + let api = Api::new(config.rpc.clone(), config.ws.clone()); + let tracker = Rc::new(RefCell::new(Tracker::default())); + let routes = (0..profile.sockets) + .map(|socket| { + accounts + .iter() + .copied() + .enumerate() + .skip(socket) + .step_by(profile.sockets) + .collect::>() + }) + .collect::>(); + let initialize = async { + tokio::try_join!( + try_join_all( + routes.iter().map(|accounts| account_socket(&config.ws, accounts, &tracker)) + ), + try_join_all((0..profile.sockets).map(|_| connect(&config.ws, &tracker))), + ) + }; + let initialized = tokio::select! { + _ = start.changed() => return Ok(None), + result = initialize => result.with_context(|| format!("initializing benchmark worker {index}")), + }; + let _ = ready.send(initialized.is_ok()).await; + drop(ready); + let (accounts_ws, signature_ws) = initialized?; + let mut tasks = JoinSet::new(); + for (socket, route) in accounts_ws.into_iter().zip(routes) { + tasks.spawn_local(restore_accounts( + config.ws.clone(), + socket, + route, + tracker.clone(), + )); + } + let signatures: Rc> = Rc::new( + signature_ws + .into_iter() + .map(|socket| Rc::new(RefCell::new(Some(socket)))) + .collect(), + ); + for (socket_index, socket) in signatures.iter().cloned().enumerate() { + tasks.spawn_local(restore_signatures( + config.ws.clone(), + socket, + socket_index, + profile.sockets, + tracker.clone(), + )); + } + tasks.spawn_local(poll_status( + config.rpc.clone(), + config.ws.clone(), + tracker.clone(), + )); + tasks.spawn_local(expire(tracker.clone(), signatures.clone())); + let Ok(clock) = start.wait_for(Option::is_some).await else { + return Ok(None); + }; + let clock = clock.expect("the coordinator published a measurement clock"); + tracker.borrow_mut().data.clock = Some(clock); + + let request_slots = Arc::new(Semaphore::new(profile.http_concurrency)); + let mut rng = random::rng(SEED, &["benchmark"], index as u64); + let mut indices: Vec<_> = (0..pool_size).collect(); + let stop = worker_stop.clone(); + let requests = execute_for_raw( + Pacing::PerSecond(profile.rate_per_worker), + profile.concurrency, + clock.end, + worker_stop, + |id| { + let account = accounts[rng.gen_range(0..pool_size)]; + let targets = rng.gen_ratio(4, 5).then(|| { + let mut mask = 0u128; + let targets: [Pubkey; WRITABLE] = std::array::from_fn(|position| { + let other = rng.gen_range(position..pool_size); + indices.swap(position, other); + let selected = indices[position]; + mask |= 1 << selected; + accounts[selected] + }); + (targets, mask) + }); + let sender = senders[rng.gen_range(0..senders.len())].clone(); + let socket = signatures[id as usize % profile.sockets].clone(); + let tracker = tracker.clone(); + let api = api.clone(); + let request_slots = request_slots.clone(); + let stop = stop.clone(); + let requested_stop = config.stop.clone(); + async move { + let transaction = if let Some((targets, mask)) = targets { + let transaction = + time::timeout(RPC_TIMEOUT, sender.prepare(&[REDLINE.write(id, &targets)])) + .await + .context("preparing benchmark transaction timed out")??; + let signature = transaction.signatures[0]; + let connection = subscribe_signature(&socket, id, &signature, &tracker).await?; + Some((transaction, signature, connection, mask)) + } else { + None + }; + let permit = + request_slots.acquire_owned().await.context("benchmark HTTP pool closed")?; + Ok(SendTransaction(move |sent: Instant| async move { + let _permit = permit; + if sent >= clock.end || *stop.borrow() || *requested_stop.borrow() { + if let Some((_, _, connection, _)) = transaction { + unsubscribe_signature(connection, id).await; + } + return Ok(()); + } + if let Some((transaction, signature, connection, mask)) = transaction { + let current = socket.borrow().as_ref().is_some_and(|current| { + Rc::ptr_eq(current, &connection) && connection.is_open() + }); + if !current { + tracker.borrow_mut().data.count(Count::WriteSkipped, sent, sent, 1); + return Err(anyhow!("signature connection closed before submission")); + } + tracker.borrow_mut().submitted(id, signature, sent, mask); + let result = + time::timeout(RPC_TIMEOUT, sender.submit_prepared(&transaction)) + .await + .context("sendTransaction timed out") + .flatten(); + tracker.borrow_mut().http_done(id, Instant::now(), result.is_ok()); + result.map(|_| ()) + } else { + let result = time::timeout(RPC_TIMEOUT, api.get_account(&account)) + .await + .context("getAccountInfo timed out") + .flatten() + .and_then(|account| account.context("benchmark account was missing")); + let ok = result.is_ok(); + tracker.borrow_mut().data.record( + sent, + Instant::now(), + Some(if ok { Count::Read } else { Count::ReadError }), + ok.then_some(Latency::Read), + ); + result.map(|_| ()) + } + })) + } + }, + ) + .await; + + let drain = time::timeout(EXPIRY + RPC_TIMEOUT, async { + while tracker.borrow().pending() { + time::sleep(POLL_INTERVAL).await; + } + }) + .await; + tasks.shutdown().await; + let result = requests.and_then(|_| drain.context("benchmark observations did not drain")); + let samples = std::mem::take(&mut tracker.borrow_mut().data.samples); + Ok(Some(Outcome { result, samples })) +} + +async fn subscribe_signature( + slot: &SocketSlot, + id: u64, + signature: &Signature, + tracker: &SharedTracker, +) -> Result> { + let started = Instant::now(); + let connection = slot.borrow().clone(); + let Some(connection) = connection else { + tracker.borrow_mut().data.count(Count::WriteSkipped, started, started, 1); + return Err(anyhow!("signature connection is reconnecting")); + }; + let ack = time::timeout(RPC_TIMEOUT, connection.signature(id, signature)).await; + let now = Instant::now(); + if !matches!(ack, Ok(Ok(()))) { + let data = &mut tracker.borrow_mut().data; + data.count(Count::AckError, started, now, 1); + data.count(Count::WriteSkipped, started, now, 1); + tokio::task::spawn_local(unsubscribe_signature(connection, id)); + return Err(anyhow!("signature subscription ACK failed: {ack:?}")); + } + tracker.borrow_mut().data.record(started, now, None, Some(Latency::Ack)); + Ok(connection) +} + +async fn connect(ws: &Url, tracker: &SharedTracker) -> Result> { + let tracker = tracker.clone(); + let socket = time::timeout( + RPC_TIMEOUT, + ObservationSocket::connect(ws, move |event| { + tracker.borrow_mut().observe(event, Instant::now()); + }), + ) + .await + .context("benchmark WebSocket connect timed out")??; + Ok(Rc::new(socket)) +} + +async fn account_socket( + ws: &Url, + accounts: &[(usize, Pubkey)], + tracker: &SharedTracker, +) -> Result> { + let socket = connect(ws, tracker).await?; + time::timeout( + RPC_TIMEOUT, + try_join_all(accounts.iter().map(|(index, account)| socket.account(*index, account))), + ) + .await + .context("benchmark account subscription timed out")??; + Ok(socket) +} + +async fn restore_accounts( + ws: Url, + mut socket: Rc, + accounts: Vec<(usize, Pubkey)>, + tracker: SharedTracker, +) { + loop { + disconnected(&tracker, socket.closed().await); + socket = reconnect(async || account_socket(&ws, &accounts, &tracker).await).await; + event(&tracker, Count::Reconnect); + } +} + +async fn restore_signatures( + ws: Url, + slot: SocketSlot, + index: usize, + sockets: usize, + tracker: SharedTracker, +) { + let mut socket = slot.borrow().clone().expect("initial signature socket"); + let mut cleanup = JoinSet::new(); + loop { + let reason = socket.closed().await; + slot.borrow_mut().take(); + disconnected(&tracker, reason); + let (replacement, restored) = + reconnect(async || signature_socket(&ws, index, sockets, &tracker).await).await; + socket = replacement; + while cleanup.try_join_next().is_some() {} + for id in restored { + if !tracker.borrow().signature_pending(id) { + cleanup.spawn_local(unsubscribe_signature(socket.clone(), id)); + } + } + slot.borrow_mut().replace(socket.clone()); + event(&tracker, Count::Reconnect); + } +} + +async fn reconnect(mut connect: impl AsyncFnMut() -> Result) -> T { + loop { + if let Ok(socket) = connect().await { + return socket; + } + time::sleep(POLL_INTERVAL).await; + } +} + +async fn signature_socket( + ws: &Url, + index: usize, + sockets: usize, + tracker: &SharedTracker, +) -> Result<(Rc, Vec)> { + let socket = connect(ws, tracker).await?; + let pending = tracker.borrow().restore(index, sockets); + let connection = &socket; + let restored = stream::iter(pending) + .map(|(id, signature)| async move { + if !tracker.borrow().signature_pending(id) { + return Ok(None); + } + time::timeout(RPC_TIMEOUT, connection.signature(id, &signature)) + .await + .context("signature restore ACK timed out")??; + Ok::<_, anyhow::Error>(Some(id)) + }) + .buffer_unordered(128) + .try_filter_map(future::ok) + .try_collect() + .await?; + Ok((socket, restored)) +} + +fn disconnected(tracker: &SharedTracker, reason: ObservationClose) { + event(tracker, Count::Disconnect); + match reason { + ObservationClose::GoingAway => event(tracker, Count::GoingAway), + ObservationClose::Transport => event(tracker, Count::TransportDisconnected), + ObservationClose::Other => {} + } +} + +fn event(tracker: &SharedTracker, counter: Count) { + let now = Instant::now(); + tracker.borrow_mut().data.count(counter, now, now, 1); +} + +async fn poll_status(rpc: Url, ws: Url, tracker: SharedTracker) { + let api = Api::new(rpc, ws); + let mut interval = time::interval(POLL_INTERVAL); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + let pending = tracker.borrow().statuses(Instant::now()); + stream::iter(pending.chunks(STATUS_BATCH)) + .for_each_concurrent(STATUS_PARALLELISM, |batch| { + let api = &api; + let tracker = &tracker; + async move { + let signatures: Vec<_> = + batch.iter().map(|(_, signature)| *signature).collect(); + let results = match time::timeout( + RPC_TIMEOUT, + api.get_signature_statuses_with_history(&signatures), + ) + .await + { + Ok(Ok(results)) if results.len() == batch.len() => results, + _ => { + event(tracker, Count::StatusRpcError); + return; + } + }; + let now = Instant::now(); + let mut tracker = tracker.borrow_mut(); + for ((id, _), result) in batch.iter().zip(results) { + if let Some(status) = result { + tracker.status(*id, status.err.is_none(), now); + } + } + } + }) + .await; + } +} + +async fn expire(tracker: SharedTracker, slots: Rc>) { + let mut interval = time::interval(POLL_INTERVAL); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut cleanup = JoinSet::new(); + loop { + interval.tick().await; + while cleanup.try_join_next().is_some() {} + for id in tracker.borrow_mut().reap(Instant::now()) { + let socket = slots[id as usize % slots.len()].borrow().clone(); + if let Some(socket) = socket { + cleanup.spawn_local(unsubscribe_signature(socket, id)); + } + } + } +} + +async fn unsubscribe_signature(socket: Rc, id: u64) { + let _ = time::timeout(RPC_TIMEOUT, socket.unsubscribe_signature(id)).await; +} diff --git a/redline/src/scenarios/aperture/mod.rs b/redline/src/scenarios/aperture/mod.rs index e96b885..17aa0e3 100644 --- a/redline/src/scenarios/aperture/mod.rs +++ b/redline/src/scenarios/aperture/mod.rs @@ -1,3 +1,4 @@ +pub mod fixed_pool; pub mod transaction_load; pub mod websocket_churn; pub mod websocket_fanout; diff --git a/redline/src/scenarios/mod.rs b/redline/src/scenarios/mod.rs index c4ae3a4..93826ca 100644 --- a/redline/src/scenarios/mod.rs +++ b/redline/src/scenarios/mod.rs @@ -3,11 +3,28 @@ pub mod harness; pub mod scheduler; use aperture::transaction_load::{Shape, TransactionLoad}; use redsuite_core::{ - catalog::{Fixture, ScenarioEntry, DEFAULT}, + catalog::{ErGroup, Fixture, ScenarioEntry, DEFAULT}, scenario, Scenario, }; + +static FIXED_POOL: ErGroup = ErGroup { + label: "fixed_pool", + settings: |params| { + params.engine.blockstore.blocktime = std::time::Duration::from_millis(50); + params.engine.blockstore.superblock = 24_000; + params.engine.ledger.size_limit = 50_000_000_000; + }, + proxy: false, + followers: 0, +}; + pub fn catalog() -> Vec { vec![ + scenario!( + aperture::fixed_pool::FixedPool, + exclusive(&FIXED_POOL, &[Fixture::RedlineProgram], ()) + ) + .explicit_only(), scenario!( TransactionLoad, shared(&DEFAULT, &[Fixture::RedlineProgram], Shape::Write)