diff --git a/Cargo.lock b/Cargo.lock index 7bdb24a9d3..4364f622c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6662,6 +6662,7 @@ checksum = "37b26c20e2178756451cfeb0661fb74c47dd5988cb7e3939de7e9241fd604d42" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", + "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-server", "jsonrpsee-types", @@ -6719,6 +6720,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "jsonrpsee-http-client" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c872b6c9961a4ccc543e321bb5b89f6b2d2c7fe8b61906918273a3333c95400c" +dependencies = [ + "async-trait", + "base64 0.22.1", + "http-body 1.0.1", + "hyper 1.7.0", + "hyper-rustls", + "hyper-util", + "jsonrpsee-core", + "jsonrpsee-types", + "rustls", + "rustls-platform-verifier", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tower", + "tracing", + "url", +] + [[package]] name = "jsonrpsee-proc-macros" version = "0.24.9" @@ -8265,6 +8291,7 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "frame-metadata-hash-extension", + "frame-support", "frame-system", "frame-system-rpc-runtime-api", "futures", diff --git a/node/Cargo.toml b/node/Cargo.toml index 2766893452..e787009d9a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -26,7 +26,7 @@ clap = { workspace = true, features = ["derive"] } futures = { workspace = true, features = ["thread-pool"] } serde = { workspace = true, features = ["derive"] } hex.workspace = true -tokio = { workspace = true, features = ["time"] } +tokio = { workspace = true, features = ["time", "rt", "net"] } # Storage import memmap2.workspace = true @@ -68,6 +68,7 @@ sp-offchain.workspace = true sp-session.workspace = true frame-metadata-hash-extension.workspace = true frame-system.workspace = true +frame-support.workspace = true pallet-transaction-payment.workspace = true pallet-commitments.workspace = true pallet-drand.workspace = true @@ -80,7 +81,7 @@ polkadot-sdk = { workspace = true, features = [ ] } # These dependencies are used for the subtensor's RPCs -jsonrpsee = { workspace = true, features = ["server"] } +jsonrpsee = { workspace = true, features = ["server", "http-client"] } sc-rpc.workspace = true sp-api.workspace = true sc-rpc-api.workspace = true @@ -156,6 +157,7 @@ runtime-benchmarks = [ "node-subtensor-runtime/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-benchmarking-cli/runtime-benchmarks", + "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", "sc-service/runtime-benchmarks", "sp-runtime/runtime-benchmarks", @@ -174,6 +176,7 @@ pow-faucet = [] # in the near future. try-runtime = [ "node-subtensor-runtime/try-runtime", + "frame-support/try-runtime", "frame-system/try-runtime", "pallet-transaction-payment/try-runtime", "sp-runtime/try-runtime", diff --git a/node/src/cli.rs b/node/src/cli.rs index e46c71857b..e7719b619c 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -8,6 +8,8 @@ use node_subtensor_runtime::opaque::Block; use sc_cli::RunCmd; use sc_consensus::BasicQueue; use sc_service::{Configuration, TaskManager}; +use std::fmt; +use std::path::PathBuf; use std::sync::Arc; #[derive(Debug, clap::Parser)] @@ -33,6 +35,14 @@ pub struct Cli { #[command(flatten)] pub eth: EthConfiguration, + + /// Control historical gap-backfill during initial/catch-up sync. + /// + /// `keep` preserves complete history (default for normal node runs). + /// `skip` is faster/lighter but historical block data may be incomplete. + /// For `build-patched-spec`, the implicit default is `skip` unless this flag is explicitly set. + #[arg(long, value_enum, default_value_t = HistoryBackfill::Keep)] + pub history_backfill: HistoryBackfill, } #[allow(clippy::large_enum_variant)] @@ -70,6 +80,93 @@ pub enum Subcommand { // Db meta columns information. ChainInfo(sc_cli::ChainInfoCmd), + + // Build a patched test chainspec from synced network state. + #[command(name = "build-patched-spec")] + CloneState(CloneStateCmd), +} + +/// Build a patched clone chainspec by syncing state, exporting raw state, and applying test patch. +#[derive(Debug, Clone, clap::Args)] +pub struct CloneStateCmd { + /// Chain spec identifier or path (same semantics as `--chain`). + #[arg(long, value_name = "CHAIN")] + pub chain: String, + + /// Base path used for syncing and state export. + #[arg(long, value_name = "PATH")] + pub base_path: PathBuf, + + /// Output file path for the final patched chainspec JSON. + #[arg(long, value_name = "FILE")] + pub output: PathBuf, + + /// Sync mode for the temporary sync node. + #[arg(long, value_enum, default_value_t = sc_cli::SyncMode::Warp)] + pub sync: sc_cli::SyncMode, + + /// Database backend for the temporary sync/export node. + #[arg(long, value_enum, default_value_t = sc_cli::Database::ParityDb)] + pub database: sc_cli::Database, + + /// RPC port used by the temporary sync node. + #[arg(long, default_value_t = 9966)] + pub rpc_port: u16, + + /// P2P port used by the temporary sync node. + #[arg(long, default_value_t = 30466)] + pub port: u16, + + /// Maximum time to wait for sync completion. + #[arg(long, default_value_t = 7200)] + pub sync_timeout_sec: u64, + + /// Accept sync completion when current is within this many blocks of highest. + #[arg(long, default_value_t = 8)] + pub sync_lag_blocks: u64, + + /// Optional bootnodes for the sync step. Repeatable. + #[arg(long, value_name = "BOOTNODE")] + pub bootnodes: Vec, + + /// Include Alice in patched validator authorities (default if no validator flags are passed; + /// Sudo is assigned to the first selected validator in Alice->Bob->Charlie order). + #[arg(long, default_value_t = false)] + pub alice: bool, + + /// Include Bob in patched validator authorities (if any validator flag is set, only selected + /// validators are used; Sudo is assigned to the first selected validator in Alice->Bob->Charlie + /// order). + #[arg(long, default_value_t = false)] + pub bob: bool, + + /// Include Charlie in patched validator authorities (if any validator flag is set, only + /// selected validators are used; Sudo is assigned to the first selected validator in + /// Alice->Bob->Charlie order). + #[arg(long, default_value_t = false)] + pub charlie: bool, +} + +#[derive(Debug, Clone, Copy, clap::ValueEnum, Default)] +pub enum HistoryBackfill { + #[default] + Keep, + Skip, +} + +impl AsRef for HistoryBackfill { + fn as_ref(&self) -> &str { + match self { + HistoryBackfill::Keep => "keep", + HistoryBackfill::Skip => "skip", + } + } +} + +impl fmt::Display for HistoryBackfill { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_ref()) + } } /// Available Sealing methods. @@ -99,6 +196,7 @@ impl SupportedConsensusMechanism { &self, config: &mut Configuration, eth_config: &EthConfiguration, + skip_history_backfill: bool, ) -> Result< ( Arc, @@ -110,8 +208,12 @@ impl SupportedConsensusMechanism { sc_service::Error, > { match self { - SupportedConsensusMechanism::Aura => new_chain_ops::(config, eth_config), - SupportedConsensusMechanism::Babe => new_chain_ops::(config, eth_config), + SupportedConsensusMechanism::Aura => { + new_chain_ops::(config, eth_config, skip_history_backfill) + } + SupportedConsensusMechanism::Babe => { + new_chain_ops::(config, eth_config, skip_history_backfill) + } } } } diff --git a/node/src/clone_spec.rs b/node/src/clone_spec.rs new file mode 100644 index 0000000000..65489d24c6 --- /dev/null +++ b/node/src/clone_spec.rs @@ -0,0 +1,616 @@ +//! Build-and-patch workflow for producing a local test chainspec from live network state. +//! +//! This module implements the `build-patched-spec` subcommand scenario: +//! +//! 1. Start a temporary node and sync it to the requested chain. +//! 2. Wait until sync is considered stable (RPC-reported near-head status). +//! 3. Stop the temporary node and run `export-state` from the synced database. +//! 4. Apply patching to the raw chainspec: +//! - replace validator/authority sets with selected dev authorities, +//! - set Sudo to the first selected validator, +//! - clear session-derived keys and localize top-level chain fields. +//! 5. Write the final patched chainspec JSON to the requested output path. +//! +//! The result is intended for local/mainnet-clone style testing where runtime state is taken from a +//! live network, but governance/validator control is reassigned to test authorities. + +use std::collections::VecDeque; +use std::fs::{self, File}; +use std::io::{BufReader, BufWriter}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use jsonrpsee::{ + core::client::ClientT, + http_client::{HttpClient, HttpClientBuilder}, + rpc_params, +}; +use serde_json::{Value, json}; +use sp_runtime::codec::Encode; + +use crate::cli::CloneStateCmd; + +type CloneResult = Result>; + +const RPC_POLL_INTERVAL: Duration = Duration::from_secs(2); +const GRANDPA_AUTHORITIES_WELL_KNOWN_KEY: &[u8] = b":grandpa_authorities"; + +/// Execute `build-patched-spec`: sync network state, export raw chainspec, apply clone patch. +pub fn run(cmd: &CloneStateCmd, skip_history_backfill: bool) -> sc_cli::Result<()> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .map_err(|err| sc_cli::Error::Application(Box::new(err)))?; + + runtime + .block_on(async_run(cmd, skip_history_backfill)) + .map_err(sc_cli::Error::Application) +} + +async fn async_run(cmd: &CloneStateCmd, skip_history_backfill: bool) -> CloneResult<()> { + let validators = selected_validators(cmd); + let selected_names = validators + .iter() + .map(|seed| seed.to_ascii_lowercase()) + .collect::>() + .join(","); + + fs::create_dir_all(&cmd.base_path)?; + + if let Some(parent) = cmd.output.parent() { + fs::create_dir_all(parent)?; + } + + let current_exe = std::env::current_exe()?; + let database_arg = database_arg(cmd.database); + let sync_arg = sync_arg(cmd.sync); + + log::info!( + "build-patched-spec: validators={} history_backfill={}", + selected_names, + if skip_history_backfill { + "skip" + } else { + "keep" + } + ); + + let mut sync_args = vec![ + "--base-path".to_string(), + cmd.base_path.display().to_string(), + "--chain".to_string(), + cmd.chain.clone(), + "--sync".to_string(), + sync_arg.to_string(), + "--database".to_string(), + database_arg.to_string(), + "--rpc-port".to_string(), + cmd.rpc_port.to_string(), + "--port".to_string(), + cmd.port.to_string(), + "--rpc-methods".to_string(), + "unsafe".to_string(), + "--no-telemetry".to_string(), + "--no-prometheus".to_string(), + "--no-mdns".to_string(), + "--name".to_string(), + "build-patched-spec-sync".to_string(), + "--history-backfill".to_string(), + if skip_history_backfill { + "skip".to_string() + } else { + "keep".to_string() + }, + ]; + + for bootnode in &cmd.bootnodes { + sync_args.push("--bootnodes".to_string()); + sync_args.push(bootnode.clone()); + } + + log::info!("build-patched-spec: starting sync node"); + + let mut sync_child = Command::new(¤t_exe) + .args(&sync_args) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + + let sync_wait_result = wait_for_sync_completion(&mut sync_child, cmd).await; + let stop_result = stop_child_gracefully(&mut sync_child).await; + + sync_wait_result?; + stop_result?; + + let raw_tmp = temp_raw_path()?; + + log::info!("build-patched-spec: exporting raw state"); + + export_raw_state(¤t_exe, cmd, database_arg, &raw_tmp)?; + + log::info!("build-patched-spec: applying clone patch"); + + patch_raw_chainspec_file(&raw_tmp, &cmd.output, &validators)?; + + if let Err(err) = fs::remove_file(&raw_tmp) { + log::warn!( + "build-patched-spec: warning: failed to remove temp file {}: {err}", + raw_tmp.display() + ); + } + + log::info!("build-patched-spec: wrote {}", cmd.output.display()); + + Ok(()) +} + +async fn wait_for_sync_completion(sync_child: &mut Child, cmd: &CloneStateCmd) -> CloneResult<()> { + let timeout = Duration::from_secs(cmd.sync_timeout_sec); + let start = Instant::now(); + let mut stable_ready_checks = 0u8; + let rpc_url = format!("http://127.0.0.1:{}", cmd.rpc_port); + let rpc_client = HttpClientBuilder::default() + .request_timeout(Duration::from_secs(10)) + .build(rpc_url)?; + + log::info!( + "build-patched-spec: waiting for sync completion (timeout={}s)", + cmd.sync_timeout_sec + ); + + while sync_child + .try_wait() + .map_err(|err| std::io::Error::other(format!("Failed to poll sync node process: {err}")))? + .is_none() + { + if start.elapsed() > timeout { + return Err(format!( + "Timed out waiting for sync completion after {} seconds", + cmd.sync_timeout_sec + ) + .into()); + } + + match query_sync_status(&rpc_client).await { + Ok(status) => { + let is_ready = !status.is_syncing + && status.peers > 0 + && status.current > 0 + && status.highest > 0 + && status.current.saturating_add(cmd.sync_lag_blocks) >= status.highest; + + if is_ready { + stable_ready_checks = stable_ready_checks.saturating_add(1); + if stable_ready_checks >= 3 { + log::info!("build-patched-spec: sync target reached"); + return Ok(()); + } + } else { + stable_ready_checks = 0; + } + } + Err(_) => { + // RPC may not be ready yet. + stable_ready_checks = 0; + } + } + + tokio::time::sleep(RPC_POLL_INTERVAL).await; + } + + let status = sync_child + .try_wait() + .map_err(|err| std::io::Error::other(format!("Failed to poll sync node process: {err}")))? + .ok_or_else(|| std::io::Error::other("Sync node status became unavailable"))?; + + Err(format!("Sync node exited unexpectedly: {status}").into()) +} + +async fn stop_child_gracefully(child: &mut Child) -> CloneResult<()> { + if child.try_wait()?.is_some() { + return Ok(()); + } + + Command::new("kill") + .arg("-INT") + .arg(child.id().to_string()) + .status()?; + + for _ in 0..30 { + if child.try_wait()?.is_some() { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + + child.kill()?; + + child.wait()?; + + Ok(()) +} + +fn export_raw_state( + current_exe: &Path, + cmd: &CloneStateCmd, + database_arg: &str, + raw_tmp: &Path, +) -> CloneResult<()> { + let stdout = File::create(raw_tmp)?; + let status = Command::new(current_exe) + .args([ + "export-state", + "--chain", + &cmd.chain, + "--base-path", + &cmd.base_path.display().to_string(), + "--database", + database_arg, + ]) + .stdin(Stdio::null()) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::inherit()) + .status()?; + + if !status.success() { + return Err(format!("export-state failed with status {status}").into()); + } + + Ok(()) +} + +struct SyncStatus { + current: u64, + highest: u64, + peers: u64, + is_syncing: bool, +} + +async fn query_sync_status(rpc_client: &HttpClient) -> CloneResult { + let sync = rpc_call(rpc_client, "system_syncState").await?; + let health = rpc_call(rpc_client, "system_health").await?; + + let current = parse_u64_field(&sync, "currentBlock") + .ok_or_else(|| "system_syncState.currentBlock missing".to_string())?; + let highest = parse_u64_field(&sync, "highestBlock") + .ok_or_else(|| "system_syncState.highestBlock missing".to_string())?; + let peers = parse_u64_field(&health, "peers") + .ok_or_else(|| "system_health.peers missing".to_string())?; + let is_syncing = health + .get("isSyncing") + .and_then(Value::as_bool) + .ok_or_else(|| "system_health.isSyncing missing".to_string())?; + + Ok(SyncStatus { + current, + highest, + peers, + is_syncing, + }) +} + +async fn rpc_call(rpc_client: &HttpClient, method: &str) -> CloneResult { + rpc_client + .request(method, rpc_params![]) + .await + .map_err(Into::into) +} + +fn parse_u64_field(value: &Value, field: &str) -> Option { + let field_value = value.get(field)?; + + if let Value::Number(number) = field_value { + return number.to_string().parse::().ok(); + } + + let s = field_value.as_str()?; + + s.parse::() + .ok() + .or_else(|| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) +} + +fn temp_raw_path() -> CloneResult { + let epoch = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + Ok(std::env::temp_dir().join(format!("subtensor-clone-export-{epoch}.json"))) +} + +fn sync_arg(mode: sc_cli::SyncMode) -> &'static str { + match mode { + sc_cli::SyncMode::Full => "full", + sc_cli::SyncMode::Fast => "fast", + sc_cli::SyncMode::FastUnsafe => "fast-unsafe", + sc_cli::SyncMode::Warp => "warp", + } +} + +fn database_arg(database: sc_cli::Database) -> &'static str { + match database { + #[cfg(feature = "rocksdb")] + sc_cli::Database::RocksDb => "rocksdb", + sc_cli::Database::ParityDb => "paritydb", + sc_cli::Database::Auto => "auto", + sc_cli::Database::ParityDbDeprecated => "paritydb-experimental", + } +} + +fn selected_validators(cmd: &CloneStateCmd) -> Vec<&'static str> { + let explicit = cmd.alice || cmd.bob || cmd.charlie; + let mut selected = Vec::new(); + + if explicit { + if cmd.alice { + selected.push("Alice"); + } + if cmd.bob { + selected.push("Bob"); + } + if cmd.charlie { + selected.push("Charlie"); + } + } else { + selected.push("Alice"); // only alice by default + } + + selected +} + +fn patch_raw_chainspec_file( + input: &Path, + output: &Path, + validators: &[&'static str], +) -> CloneResult<()> { + let file = File::open(input)?; + let reader = BufReader::with_capacity(64 * 1024 * 1024, file); + let mut spec: Value = serde_json::from_reader(reader)?; + patch_raw_spec(&mut spec, validators)?; + + let out = File::create(output)?; + let writer = BufWriter::with_capacity(64 * 1024 * 1024, out); + serde_json::to_writer(writer, &spec)?; + Ok(()) +} + +fn patch_raw_spec(spec: &mut Value, validators: &[&'static str]) -> CloneResult<()> { + let sudo = validators + .first() + .ok_or_else(|| "at least one validator must be selected".to_string())?; + + let top = spec + .pointer_mut("/genesis/raw/top") + .and_then(Value::as_object_mut) + .ok_or_else(|| "missing or invalid genesis.raw.top".to_string())?; + + let aura_keys = validators + .iter() + .map(|seed| crate::chain_spec::authority_keys_from_seed(seed).0) + .collect::>(); + top.insert( + storage_key("Aura", "Authorities"), + Value::String(to_hex(&aura_keys.encode())), + ); + + let grandpa_entries = validators + .iter() + .map(|seed| (crate::chain_spec::authority_keys_from_seed(seed).1, 1u64)) + .collect::>(); + let grandpa_encoded = grandpa_entries.encode(); + + top.insert( + storage_key("Grandpa", "Authorities"), + Value::String(to_hex(&grandpa_encoded)), + ); + + let mut well_known = vec![0x01u8]; + well_known.extend_from_slice(&grandpa_encoded); + top.insert( + to_hex(GRANDPA_AUTHORITIES_WELL_KNOWN_KEY), + Value::String(to_hex(&well_known)), + ); + + top.insert( + storage_key("Grandpa", "CurrentSetId"), + Value::String(to_hex(&0u64.to_le_bytes())), + ); + top.insert( + storage_key("Grandpa", "State"), + Value::String("0x00".into()), + ); + top.remove(&storage_key("Grandpa", "PendingChange")); + top.remove(&storage_key("Grandpa", "NextForced")); + top.remove(&storage_key("Grandpa", "Stalled")); + remove_by_prefix(top, &storage_key("Grandpa", "SetIdSession")); + + top.insert( + storage_key("Sudo", "Key"), + Value::String(to_hex( + &crate::chain_spec::get_account_id_from_seed::(sudo).encode(), + )), + ); + + remove_by_prefix(top, &storage_prefix("Session")); + clear_top_level(spec); + Ok(()) +} + +fn remove_by_prefix(map: &mut serde_json::Map, prefix: &str) { + let mut keys_to_remove = VecDeque::new(); + for key in map.keys() { + if key.starts_with(prefix) { + keys_to_remove.push_back(key.clone()); + } + } + while let Some(key) = keys_to_remove.pop_front() { + map.remove(&key); + } +} + +fn clear_top_level(spec: &mut Value) { + if let Some(object) = spec.as_object_mut() { + object.insert("bootNodes".into(), json!([])); + object.insert("codeSubstitutes".into(), json!({})); + object.insert("chainType".into(), json!("Local")); + } +} + +fn storage_key(pallet: &str, item: &str) -> String { + let key = frame_support::storage::storage_prefix(pallet.as_bytes(), item.as_bytes()); + to_hex(&key) +} + +fn storage_prefix(pallet: &str) -> String { + format!( + "0x{}", + hex::encode(sp_io::hashing::twox_128(pallet.as_bytes())) + ) +} + +fn to_hex(data: &[u8]) -> String { + format!("0x{}", hex::encode(data)) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + fn target_artifact_path(name: &str) -> PathBuf { + let target_dir = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target")); + target_dir.join("clone-spec-tests").join(name) + } + + fn default_cmd() -> CloneStateCmd { + CloneStateCmd { + chain: "finney".to_string(), + base_path: target_artifact_path("base"), + output: target_artifact_path("out.json"), + sync: sc_cli::SyncMode::Warp, + database: sc_cli::Database::ParityDb, + rpc_port: 9966, + port: 30466, + sync_timeout_sec: 10, + sync_lag_blocks: 8, + bootnodes: Vec::new(), + alice: false, + bob: false, + charlie: false, + } + } + + fn make_minimal_spec() -> Value { + let mut top = serde_json::Map::new(); + top.insert(storage_key("Grandpa", "PendingChange"), json!("0x01")); + top.insert(storage_key("Grandpa", "NextForced"), json!("0x02")); + top.insert(storage_key("Grandpa", "Stalled"), json!("0x03")); + top.insert( + format!("{}{}", storage_key("Grandpa", "SetIdSession"), "deadbeef"), + json!("0x04"), + ); + top.insert(format!("{}abcd", storage_prefix("Session")), json!("0x05")); + top.insert(storage_key("Balances", "TotalIssuance"), json!("0x06")); + + json!({ + "genesis": { "raw": { "top": top } }, + "bootNodes": ["/dns4/example.com/tcp/30333/p2p/12D3KooW..."], + "codeSubstitutes": { "0x01": "0x02" }, + "chainType": "Live" + }) + } + + #[test] + fn selected_validators_defaults_to_alice() { + let cmd = default_cmd(); + let selected = selected_validators(&cmd); + assert_eq!(selected.len(), 1); + assert_eq!(selected.first(), Some(&"Alice")); + } + + #[test] + fn selected_validators_respects_explicit_flags() { + let mut cmd = default_cmd(); + cmd.bob = true; + cmd.charlie = true; + + let selected = selected_validators(&cmd); + assert_eq!(selected, vec!["Bob", "Charlie"]); + } + + #[test] + fn parse_u64_field_supports_u64_decimal_and_hex_string() { + let value = json!({ + "a": 42, + "b": "123", + "c": "0x2a" + }); + + assert_eq!(parse_u64_field(&value, "a"), Some(42)); + assert_eq!(parse_u64_field(&value, "b"), Some(123)); + assert_eq!(parse_u64_field(&value, "c"), Some(42)); + assert_eq!(parse_u64_field(&value, "missing"), None); + } + + #[test] + fn patch_raw_spec_updates_authorities_sudo_and_top_level() { + let mut spec = make_minimal_spec(); + let validators = vec!["Alice", "Bob"]; + patch_raw_spec(&mut spec, &validators).expect("patch should succeed"); + + let top = spec + .pointer("/genesis/raw/top") + .and_then(Value::as_object) + .expect("top should be object"); + + let aura_hex = top + .get(&storage_key("Aura", "Authorities")) + .and_then(Value::as_str) + .expect("aura authorities key should exist"); + let aura_raw = hex::decode(aura_hex.trim_start_matches("0x")).expect("hex decode aura"); + let expected_aura = vec![ + crate::chain_spec::authority_keys_from_seed("Alice").0, + crate::chain_spec::authority_keys_from_seed("Bob").0, + ] + .encode(); + assert_eq!(aura_raw, expected_aura); + + let sudo_hex = top + .get(&storage_key("Sudo", "Key")) + .and_then(Value::as_str) + .expect("sudo key should exist"); + assert_eq!( + sudo_hex, + to_hex( + &crate::chain_spec::get_account_id_from_seed::("Alice") + .encode() + ) + .as_str() + ); + + assert!(!top.contains_key(&storage_key("Grandpa", "PendingChange"))); + assert!(!top.contains_key(&storage_key("Grandpa", "NextForced"))); + assert!(!top.contains_key(&storage_key("Grandpa", "Stalled"))); + assert!( + top.keys() + .all(|k| !k.starts_with(&storage_prefix("Session"))) + ); + + assert_eq!(spec.get("chainType"), Some(&json!("Local"))); + assert_eq!(spec.get("bootNodes"), Some(&json!([]))); + assert_eq!(spec.get("codeSubstitutes"), Some(&json!({}))); + } + + #[test] + fn patch_raw_spec_fails_when_top_missing() { + let mut spec = json!({}); + let err = patch_raw_spec(&mut spec, &["Alice"]).expect_err("must fail"); + assert!( + err.to_string() + .contains("missing or invalid genesis.raw.top"), + "unexpected error: {err}" + ); + } +} diff --git a/node/src/command.rs b/node/src/command.rs index 670ae6f4e4..fd3a122ec5 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, atomic::AtomicBool}; use crate::{ chain_spec, - cli::{Cli, Subcommand, SupportedConsensusMechanism}, + cli::{Cli, HistoryBackfill, Subcommand, SupportedConsensusMechanism}, consensus::BabeConsensus, ethereum::db_config_dir, service, @@ -62,6 +62,7 @@ pub fn run() -> sc_cli::Result<()> { let cmd = Cli::command(); let arg_matches = cmd.get_matches(); let cli = Cli::from_arg_matches(&arg_matches)?; + let skip_history_backfill = resolve_skip_history_backfill(&cli, &arg_matches); match &cli.subcommand { Some(Subcommand::Key(cmd)) => cmd.run(&cli), @@ -72,32 +73,40 @@ pub fn run() -> sc_cli::Result<()> { Some(Subcommand::CheckBlock(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|mut config| { - let (client, _, import_queue, task_manager, _) = - cli.initial_consensus.new_chain_ops(&mut config, &cli.eth)?; + let (client, _, import_queue, task_manager, _) = cli + .initial_consensus + .new_chain_ops(&mut config, &cli.eth, skip_history_backfill)?; Ok((cmd.run(client, import_queue), task_manager)) }) } Some(Subcommand::ExportBlocks(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|mut config| { - let (client, _, _, task_manager, _) = - cli.initial_consensus.new_chain_ops(&mut config, &cli.eth)?; + let (client, _, _, task_manager, _) = cli.initial_consensus.new_chain_ops( + &mut config, + &cli.eth, + skip_history_backfill, + )?; Ok((cmd.run(client, config.database), task_manager)) }) } Some(Subcommand::ExportState(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|mut config| { - let (client, _, _, task_manager, _) = - cli.initial_consensus.new_chain_ops(&mut config, &cli.eth)?; + let (client, _, _, task_manager, _) = cli.initial_consensus.new_chain_ops( + &mut config, + &cli.eth, + skip_history_backfill, + )?; Ok((cmd.run(client, config.chain_spec), task_manager)) }) } Some(Subcommand::ImportBlocks(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|mut config| { - let (client, _, import_queue, task_manager, _) = - cli.initial_consensus.new_chain_ops(&mut config, &cli.eth)?; + let (client, _, import_queue, task_manager, _) = cli + .initial_consensus + .new_chain_ops(&mut config, &cli.eth, skip_history_backfill)?; Ok((cmd.run(client, import_queue), task_manager)) }) } @@ -150,8 +159,11 @@ pub fn run() -> sc_cli::Result<()> { Some(Subcommand::Revert(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|mut config| { - let (client, backend, _, task_manager, _) = - cli.initial_consensus.new_chain_ops(&mut config, &cli.eth)?; + let (client, backend, _, task_manager, _) = cli.initial_consensus.new_chain_ops( + &mut config, + &cli.eth, + skip_history_backfill, + )?; let aux_revert = Box::new(move |client, _, blocks| { sc_consensus_grandpa::revert(client, blocks)?; Ok(()) @@ -233,25 +245,54 @@ pub fn run() -> sc_cli::Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| cmd.run::(&config)) } + Some(Subcommand::CloneState(cmd)) => { + let runner = cli.create_runner(&cli.run)?; + let cmd = cmd.clone(); + runner.sync_run(move |_| crate::clone_spec::run(&cmd, skip_history_backfill)) + } // Start with the initial consensus type asked. - None => { - let arg_matches = Cli::command().get_matches(); - let cli = Cli::from_args(); - match cli.initial_consensus { - SupportedConsensusMechanism::Babe => start_babe_service(&arg_matches), - SupportedConsensusMechanism::Aura => start_aura_service(&arg_matches), + None => match cli.initial_consensus { + SupportedConsensusMechanism::Babe => { + start_babe_service(&arg_matches, skip_history_backfill) } - } + SupportedConsensusMechanism::Aura => { + start_aura_service(&arg_matches, skip_history_backfill) + } + }, } } +fn resolve_skip_history_backfill(cli: &Cli, arg_matches: &ArgMatches) -> bool { + // We keep a single global `--history-backfill` flag, but `build-patched-spec` should default to + // `skip` when the operator didn't set the flag explicitly. This preserves `keep` as the default + // for normal node runs. + if matches!( + arg_matches.value_source("history_backfill"), + Some(ValueSource::CommandLine) + ) { + return matches!(cli.history_backfill, HistoryBackfill::Skip); + } + + matches!(&cli.subcommand, Some(Subcommand::CloneState(_))) +} + #[allow(clippy::expect_used)] -fn start_babe_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { +fn start_babe_service( + arg_matches: &ArgMatches, + skip_history_backfill: bool, +) -> Result<(), sc_cli::Error> { let cli = Cli::from_arg_matches(arg_matches).expect("Bad arg_matches"); let runner = cli.create_runner(&cli.run)?; match runner.run_node_until_exit(|config| async move { let config = customise_config(arg_matches, config); - service::build_full::(config, cli.eth, cli.sealing, None).await + service::build_full::( + config, + cli.eth, + cli.sealing, + None, + skip_history_backfill, + ) + .await }) { Ok(_) => Ok(()), Err(e) => { @@ -264,7 +305,7 @@ fn start_babe_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { log::info!( "💡 Chain is using Aura consensus. Switching to Aura service until Babe block is detected.", ); - start_aura_service(arg_matches) + start_aura_service(arg_matches, skip_history_backfill) // Handle Aura service still has DB lock. This never has been observed to take more // than 1s to drop. } else if matches!(e, sc_service::Error::Client(sp_blockchain::Error::Backend(ref msg)) @@ -272,7 +313,7 @@ fn start_babe_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { { log::info!("Failed to aquire DB lock, trying again in 1s..."); std::thread::sleep(std::time::Duration::from_secs(1)); - start_babe_service(arg_matches) + start_babe_service(arg_matches, skip_history_backfill) // Unknown error, return it. } else { log::error!("Failed to start Babe service: {e:?}"); @@ -283,7 +324,10 @@ fn start_babe_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { } #[allow(clippy::expect_used)] -fn start_aura_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { +fn start_aura_service( + arg_matches: &ArgMatches, + skip_history_backfill: bool, +) -> Result<(), sc_cli::Error> { let cli = Cli::from_arg_matches(arg_matches).expect("Bad arg_matches"); let runner = cli.create_runner(&cli.run)?; @@ -301,13 +345,14 @@ fn start_aura_service(arg_matches: &ArgMatches) -> Result<(), sc_cli::Error> { cli.eth, cli.sealing, Some(custom_service_signal_clone), + skip_history_backfill, ) .await }) { Ok(()) => Ok(()), Err(e) => { if custom_service_signal.load(std::sync::atomic::Ordering::Relaxed) { - start_babe_service(arg_matches) + start_babe_service(arg_matches, skip_history_backfill) } else { Err(e.into()) } diff --git a/node/src/conditional_evm_block_import.rs b/node/src/conditional_evm_block_import.rs index b6ba445c1f..0a69bdc090 100644 --- a/node/src/conditional_evm_block_import.rs +++ b/node/src/conditional_evm_block_import.rs @@ -1,11 +1,12 @@ use sc_consensus::{BlockCheckParams, BlockImport, BlockImportParams, ImportResult}; -use sp_consensus::Error as ConsensusError; +use sp_consensus::{BlockOrigin, Error as ConsensusError}; use sp_runtime::traits::{Block as BlockT, Header}; use std::marker::PhantomData; pub struct ConditionalEVMBlockImport { inner: I, frontier_block_import: F, + skip_history_backfill: bool, _marker: PhantomData, } @@ -19,6 +20,7 @@ where ConditionalEVMBlockImport { inner: self.inner.clone(), frontier_block_import: self.frontier_block_import.clone(), + skip_history_backfill: self.skip_history_backfill, _marker: PhantomData, } } @@ -32,10 +34,11 @@ where F: BlockImport, F::Error: Into, { - pub fn new(inner: I, frontier_block_import: F) -> Self { + pub fn new(inner: I, frontier_block_import: F, skip_history_backfill: bool) -> Self { Self { inner, frontier_block_import, + skip_history_backfill, _marker: PhantomData, } } @@ -56,7 +59,17 @@ where self.inner.check_block(block).await.map_err(Into::into) } - async fn import_block(&self, block: BlockImportParams) -> Result { + async fn import_block( + &self, + mut block: BlockImportParams, + ) -> Result { + if self.skip_history_backfill && matches!(block.origin, BlockOrigin::NetworkInitialSync) { + // During initial network sync, Substrate can mark missing historical ranges as "gaps" + // (`create_gap = true`) and then backfill them later. When history backfill is set to + // `skip`, we disable gap creation so no history reconstruction work is scheduled. + // `build-patched-spec` just defaults this setting to `skip`. + block.create_gap = false; + } // 4345556 - mainnet runtime upgrade block with Frontier if *block.header.number() < 4345557u32.into() { self.inner.import_block(block).await.map_err(Into::into) diff --git a/node/src/consensus/aura_consensus.rs b/node/src/consensus/aura_consensus.rs index ce34e8125a..74ec8fea1e 100644 --- a/node/src/consensus/aura_consensus.rs +++ b/node/src/consensus/aura_consensus.rs @@ -139,7 +139,7 @@ impl ConsensusMechanism for AuraConsensus { Self {} } - fn build_biq(&mut self) -> Result, sc_service::Error> + fn build_biq(&mut self, skip_history_backfill: bool) -> Result, sc_service::Error> where NumberFor: BlockNumberOps, { @@ -157,6 +157,7 @@ impl ConsensusMechanism for AuraConsensus { client.clone(), grandpa_block_import.clone(), expected_babe_config.clone(), + skip_history_backfill, ); let slot_duration = sc_consensus_aura::slot_duration(&*client)?; diff --git a/node/src/consensus/babe_consensus.rs b/node/src/consensus/babe_consensus.rs index 4f84cbb87b..fad204fb48 100644 --- a/node/src/consensus/babe_consensus.rs +++ b/node/src/consensus/babe_consensus.rs @@ -152,7 +152,7 @@ impl ConsensusMechanism for BabeConsensus { } } - fn build_biq(&mut self) -> Result, sc_service::Error> + fn build_biq(&mut self, skip_history_backfill: bool) -> Result, sc_service::Error> where NumberFor: BlockNumberOps, { @@ -188,6 +188,7 @@ impl ConsensusMechanism for BabeConsensus { let conditional_block_import = ConditionalEVMBlockImport::new( babe_import.clone(), FrontierBlockImport::new(babe_import.clone(), client.clone()), + skip_history_backfill, ); let slot_duration = babe_link.config().slot_duration(); diff --git a/node/src/consensus/consensus_mechanism.rs b/node/src/consensus/consensus_mechanism.rs index 9fd8cad63b..41cb2fb4a8 100644 --- a/node/src/consensus/consensus_mechanism.rs +++ b/node/src/consensus/consensus_mechanism.rs @@ -78,7 +78,7 @@ pub trait ConsensusMechanism { fn new() -> Self; /// Builds a `BIQ` that uses the ConsensusMechanism. - fn build_biq(&mut self) -> Result, sc_service::Error>; + fn build_biq(&mut self, skip_history_backfill: bool) -> Result, sc_service::Error>; /// Returns the slot duration. fn slot_duration(&self, client: &FullClient) -> Result; diff --git a/node/src/consensus/hybrid_import_queue.rs b/node/src/consensus/hybrid_import_queue.rs index 30d8ff4065..342d67dbc1 100644 --- a/node/src/consensus/hybrid_import_queue.rs +++ b/node/src/consensus/hybrid_import_queue.rs @@ -71,10 +71,12 @@ impl HybridBlockImport { client: Arc, grandpa_block_import: GrandpaBlockImport, babe_config: BabeConfiguration, + skip_history_backfill: bool, ) -> Self { let inner_aura = ConditionalEVMBlockImport::new( grandpa_block_import.clone(), FrontierBlockImport::new(grandpa_block_import.clone(), client.clone()), + skip_history_backfill, ); #[allow(clippy::expect_used)] @@ -88,6 +90,7 @@ impl HybridBlockImport { let inner_babe = ConditionalEVMBlockImport::new( babe_import.clone(), FrontierBlockImport::new(babe_import.clone(), client.clone()), + skip_history_backfill, ); HybridBlockImport { diff --git a/node/src/lib.rs b/node/src/lib.rs index c447a07309..4740155f5e 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -1,6 +1,7 @@ pub mod chain_spec; pub mod cli; pub mod client; +pub mod clone_spec; pub mod conditional_evm_block_import; pub mod consensus; pub mod ethereum; diff --git a/node/src/main.rs b/node/src/main.rs index 64f25acc67..2766b93054 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -6,6 +6,7 @@ mod benchmarking; mod chain_spec; mod cli; mod client; +mod clone_spec; mod command; mod conditional_evm_block_import; mod consensus; diff --git a/node/src/service.rs b/node/src/service.rs index f33931d210..7a745b6c3f 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -241,6 +241,7 @@ pub fn build_manual_seal_import_queue( crate::conditional_evm_block_import::ConditionalEVMBlockImport::new( grandpa_block_import.clone(), fc_consensus::FrontierBlockImport::new(grandpa_block_import.clone(), client.clone()), + false, ); Ok(( sc_consensus_manual_seal::import_queue( @@ -259,6 +260,7 @@ pub async fn new_full( eth_config: EthConfiguration, sealing: Option, custom_service_signal: Option>, + skip_history_backfill: bool, ) -> Result where NumberFor: BlockNumberOps, @@ -275,7 +277,7 @@ where } let mut consensus_mechanism = CM::new(); - let build_import_queue = consensus_mechanism.build_biq()?; + let build_import_queue = consensus_mechanism.build_biq(skip_history_backfill)?; let PartialComponents { client, @@ -660,6 +662,7 @@ pub async fn build_full( eth_config: EthConfiguration, sealing: Option, custom_service_signal: Option>, + skip_history_backfill: bool, ) -> Result { match config.network.network_backend { sc_network::config::NetworkBackendType::Libp2p => { @@ -668,6 +671,7 @@ pub async fn build_full( eth_config, sealing, custom_service_signal, + skip_history_backfill, ) .await } @@ -677,6 +681,7 @@ pub async fn build_full( eth_config, sealing, custom_service_signal, + skip_history_backfill, ) .await } @@ -686,6 +691,7 @@ pub async fn build_full( pub fn new_chain_ops( config: &mut Configuration, eth_config: &EthConfiguration, + skip_history_backfill: bool, ) -> Result< ( Arc, @@ -705,7 +711,11 @@ pub fn new_chain_ops( task_manager, other, .. - } = new_partial(config, eth_config, consensus_mechanism.build_biq()?)?; + } = new_partial( + config, + eth_config, + consensus_mechanism.build_biq(skip_history_backfill)?, + )?; Ok((client, backend, import_queue, task_manager, other.3)) } diff --git a/pallets/subtensor/src/macros/dispatches.rs b/pallets/subtensor/src/macros/dispatches.rs index 19a8f635dc..c53b5a1570 100644 --- a/pallets/subtensor/src/macros/dispatches.rs +++ b/pallets/subtensor/src/macros/dispatches.rs @@ -1936,7 +1936,7 @@ mod dispatches { /// Emits a `TokensRecycled` event on success. #[pallet::call_index(101)] #[pallet::weight(( - Weight::from_parts(113_400_000, 0).saturating_add(T::DbWeight::get().reads_writes(7, 4)), + Weight::from_parts(113_400_000, 0).saturating_add(T::DbWeight::get().reads_writes(7_u64, 4)), DispatchClass::Normal, Pays::Yes ))] @@ -1961,7 +1961,7 @@ mod dispatches { /// Emits a `TokensBurned` event on success. #[pallet::call_index(102)] #[pallet::weight(( - Weight::from_parts(112_200_000, 0).saturating_add(T::DbWeight::get().reads_writes(7, 3)), + Weight::from_parts(112_200_000, 0).saturating_add(T::DbWeight::get().reads_writes(7_u64, 3)), DispatchClass::Normal, Pays::Yes ))] @@ -2461,7 +2461,7 @@ mod dispatches { /// #[pallet::call_index(127)] #[pallet::weight( - Weight::from_parts(20_750_000, 0) + Weight::from_parts(30_810_000, 0) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) )] diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 5e3436a78c..4bbb6a88c8 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -268,7 +268,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // `spec_version`, and `authoring_version` are the same between Wasm and native. // This value is set to 100 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use // the compatible custom types. - spec_version: 390, + spec_version: 391, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1,