feat: ratelimiting and other ux improvements (#4)

* 50k conns?

* rm: json parsing on testserver

* huge backlog

* allow get

* log concurrent

* limit to 1 worker

* 50 workers

* feat: hyper

* fix?

* actix -> axum

* Update main.rs

* rm batch factor

* 3 second measurement interval

* Update main.rs

* worker id

* request durr

* fix: workers

* Update network.rs

* fix ids

* feat: implied total rps

* Update network.rs

* Update network.rs

* Create genesis.json

* genalloc

* 25k

* start reth

* 50k again?

* wip: try one of 10k addrs

* even workers

* 20k conns

* Update main.rs

* back to 20 acc

* 100 accounts

* 100 acc

* 2500 accounts

* oops fix tx queue reporting

* feat: rand addr

* 500 accounts

* use existing people, log

* allow dead code

* Update tx_gen.rs

* lawging

* hrmf

* 1k accs

* 10k accounts

* wip: startup delay

* Update tx_queue.rs

* Update tx_queue.rs

* log

* moar thresholds

* Update tx_queue.rs

* feat: rayonify

* Update tx_gen.rs

* 50k connections?

* Revert "50k connections?"

This reverts commit 34ec48c.

* log rayon thread pool size

* Update tx_gen.rs

* Initialize Rayon with explicit thread count.

* Update tx_gen.rs

* Update tx_gen.rs

* feat: alternate ratelimit system

* Update tx_queue.rs

* Update tx_queue.rs

* Update tx_queue.rs

* Update tx_queue.rs

* Update tx_queue.rs

* hm

* thresholds

* Update tx_queue.rs

* Update tx_queue.rs

* Update tx_queue.rs

* thresholds

* Update tx_queue.rs

* fewer socket conns

* 10k connections

* 50k RPS?

* feat: longer ramp

* 35k rps?

* Update tx_queue.rs

* 25k

* Update start-reth.sh

* Update genesis.json

* Update start-reth.sh

* batch factor of 5

* fix: implied rps

* disable batching

* feat: even slower ramp up

* slightly faster

* feat: 50k genesis

* feat: better ramp

* oops

* worker ratio

* feat: pareto distribute recipients

* Update tx_gen.rs

* Empty the rate limiter.

* feat: empty rate limiter before AND after

* wip: can we remove first set avail?

* 15k tps

* 20k?

* 15k again

* feat: ramp up faster

* even faster

* faster!

Author: transmissions11

Cargo.lock

@@ -19,4 +19,11 @@ alloy = { version = "1", features = ["genesis", "signers"] }
 alloy-evm = "0.13"
 alloy-consensus = { version = "1", features = ["secp256k1"] }
 alloy-signer-local = { version = "1", features = ["mnemonic"] }
-core_affinity = "0.8.3"
+core_affinity = "0.8.3"
+rayon = "1.10"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+hex = "0.4"
+simple-tqdm = { version = "0.2.0", features = ["rayon"] }
+rand = "0.9.1"
+ratelimit = "0.10"

crescendo/Cargo.toml

@@ -0,0 +1,44 @@
+use std::collections::BTreeMap;
+use std::fs;
+use std::path::Path;
+
+use alloy::signers::local::MnemonicBuilder;
+use alloy::signers::utils::secret_key_to_address;
+use alloy_signer_local::coins_bip39::English;
+use rayon::prelude::*;
+use serde::{Deserialize, Serialize};
+use simple_tqdm::ParTqdm;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct AccountBalance {
+    balance: String,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    const NUM_ACCOUNTS: u32 = 50_000;
+    const MNEMONIC: &str = "test test test test test test test test test test test junk";
+
+    println!("Generating {} accounts...", NUM_ACCOUNTS);
+
+    let genesis_alloc: BTreeMap<String, AccountBalance> = (0..NUM_ACCOUNTS)
+        .into_par_iter()
+        .tqdm()
+        .map(|worker_id| {
+            let signer =
+                MnemonicBuilder::<English>::default().phrase(MNEMONIC).index(worker_id).unwrap().build().unwrap();
+
+            let address = secret_key_to_address(&signer.credential());
+
+            (format!("{:?}", address), AccountBalance { balance: "0xD3C21BCECCEDA1000000".to_string() })
+        })
+        .collect();
+
+    let output_path = Path::new("genesis-alloc.json");
+    let json = serde_json::to_string_pretty(&genesis_alloc)?;
+    fs::write(output_path, json)?;
+
+    println!("\nSuccessfully generated {} accounts!", NUM_ACCOUNTS);
+    println!("Accounts saved to: {}", output_path.display());
+
+    Ok(())
+}

crescendo/src/bin/generate_genesis_alloc.rs

@@ -20,7 +20,7 @@ use crate::workers::{DesireType, WorkerType};
 static GLOBAL: MiMalloc = MiMalloc;
 
 // TODO: Configurable CLI args.
-const TOTAL_CONNECTIONS: u64 = 50_000; // This is limited by the amount of ephemeral ports available on the system.
+const TOTAL_CONNECTIONS: u64 = 10_000; // This is limited by the amount of ephemeral ports available on the system.
 const THREAD_PINNING: bool = true;
 const TARGET_URL: &str = "http://127.0.0.1:8545";
 
@@ -33,13 +33,16 @@ async fn main() {
     let mut core_ids = core_affinity::get_core_ids().unwrap();
     println!("[*] Detected {} effective cores.", core_ids.len());
 
+    // Initialize Rayon with explicit thread count.
+    rayon::ThreadPoolBuilder::new().num_threads(core_ids.len()).build_global().unwrap();
+
     // Pin the tokio runtime to a core (if enabled).
     utils::maybe_pin_thread(core_ids.pop().unwrap(), THREAD_PINNING);
 
     // Given our desired breakdown of workers, translate this into actual numbers of workers to spawn.
     let (workers, worker_counts) = workers::assign_workers(
         core_ids, // Doesn't include the main runtime core.
-        vec![(WorkerType::TxGen, DesireType::Exact(20)), (WorkerType::Network, DesireType::Percentage(1.0))],
+        vec![(WorkerType::TxGen, DesireType::Percentage(0.1)), (WorkerType::Network, DesireType::Percentage(0.9))],
         THREAD_PINNING, // Only log core ranges if thread pinning is actually enabled.
     );