diff --git a/Cargo.lock b/Cargo.lock index d993de0358f7..4ac189c29f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6459,12 +6459,18 @@ version = "0.9.18" dependencies = [ "beefy-primitives", "frame-benchmarking", + "frame-benchmarking-cli", + "frame-system", "frame-system-rpc-runtime-api", "kusama-runtime", "pallet-mmr-primitives", + "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", + "polkadot-core-primitives", + "polkadot-node-core-parachains-inherent", "polkadot-primitives", "polkadot-runtime", + "polkadot-runtime-common", "rococo-runtime", "sc-client-api", "sc-consensus", @@ -6476,11 +6482,15 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-consensus-babe", + "sp-core", "sp-finality-grandpa", + "sp-inherents", + "sp-keyring", "sp-offchain", "sp-runtime", "sp-session", "sp-storage", + "sp-timestamp", "sp-transaction-pool", "westend-runtime", ] diff --git a/cli/src/cli.rs b/cli/src/cli.rs index ca34fd5ab19f..bcd2b05388d4 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -61,6 +61,13 @@ pub enum Subcommand { )] BenchmarkBlock(frame_benchmarking_cli::BlockCmd), + /// Sub command for benchmarking the per-block and per-extrinsic execution overhead. + #[clap( + name = "benchmark-overhead", + about = "Benchmark the per-block and per-extrinsic execution overhead." + )] + BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd), + /// Sub command for benchmarking the storage speed. #[clap(name = "benchmark-storage", about = "Benchmark storage speed.")] BenchmarkStorage(frame_benchmarking_cli::StorageCmd), diff --git a/cli/src/command.rs b/cli/src/command.rs index 5f995e681f3c..e3945e784f0d 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -18,11 +18,11 @@ use crate::cli::{Cli, Subcommand}; use futures::future::TryFutureExt; use log::info; use sc_cli::{Role, RuntimeVersion, SubstrateCli}; -use service::{self, IdentifyVariant}; +use service::{self, HeaderBackend, IdentifyVariant}; use sp_core::crypto::Ss58AddressFormatRegistry; use std::net::ToSocketAddrs; -pub use crate::error::Error; +pub use crate::{error::Error, service::BlockId}; pub use polkadot_performance_test::PerfCheckError; impl std::convert::From for Error { @@ -534,6 +534,101 @@ pub fn run() -> Result<()> { #[cfg(not(feature = "polkadot-native"))] unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled") }, + Some(Subcommand::BenchmarkOverhead(cmd)) => { + use polkadot_client::benchmark_inherent_data; + + let runner = cli.create_runner(cmd)?; + let chain_spec = &runner.config().chain_spec; + set_default_ss58_version(chain_spec); + ensure_dev(chain_spec).map_err(Error::Other)?; + + #[cfg(feature = "rococo-native")] + if chain_spec.is_rococo() || chain_spec.is_wococo() || chain_spec.is_versi() { + return Ok(runner.async_run(|mut config| { + let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?; + + let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap(); + let inherent_data = benchmark_inherent_data(header) + .map_err(|e| format!("generating inherent data: {:?}", e))?; + + if let polkadot_client::Client::Rococo(pd) = &*client { + Ok(( + cmd.run(config, pd.clone(), inherent_data, client) + .map_err(Error::SubstrateCli), + task_manager, + )) + } else { + unreachable!("Checked above; qed") + } + })?) + } + + #[cfg(feature = "kusama-native")] + if chain_spec.is_kusama() { + return Ok(runner.async_run(|mut config| { + let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?; + + let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap(); + let inherent_data = benchmark_inherent_data(header) + .map_err(|e| format!("generating inherent data: {:?}", e))?; + + if let polkadot_client::Client::Kusama(pd) = &*client { + Ok(( + cmd.run(config, pd.clone(), inherent_data, client) + .map_err(Error::SubstrateCli), + task_manager, + )) + } else { + unreachable!("Checked above; qed") + } + })?) + } + + #[cfg(feature = "westend-native")] + if chain_spec.is_westend() { + return Ok(runner.async_run(|mut config| { + let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?; + + let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap(); + let inherent_data = benchmark_inherent_data(header) + .map_err(|e| format!("generating inherent data: {:?}", e))?; + + if let polkadot_client::Client::Westend(pd) = &*client { + Ok(( + cmd.run(config, pd.clone(), inherent_data, client) + .map_err(Error::SubstrateCli), + task_manager, + )) + } else { + unreachable!("Checked above; qed") + } + })?) + } + + #[cfg(feature = "polkadot-native")] + { + return Ok(runner.async_run(|mut config| { + let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?; + + let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap(); + let inherent_data = benchmark_inherent_data(header) + .map_err(|e| format!("generating inherent data: {:?}", e))?; + + if let polkadot_client::Client::Polkadot(pd) = &*client { + Ok(( + cmd.run(config, pd.clone(), inherent_data, client) + .map_err(Error::SubstrateCli), + task_manager, + )) + } else { + unreachable!("Checked above; qed") + } + })?) + } + + #[cfg(not(feature = "polkadot-native"))] + unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled") + }, Some(Subcommand::BenchmarkStorage(cmd)) => { let runner = cli.create_runner(cmd)?; let chain_spec = &runner.config().chain_spec; diff --git a/node/client/Cargo.toml b/node/client/Cargo.toml index b0eac0d72f7a..869612c64f6d 100644 --- a/node/client/Cargo.toml +++ b/node/client/Cargo.toml @@ -6,7 +6,10 @@ edition = "2021" [dependencies] frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master" } +frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master" } +pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", branch = "master" } pallet-transaction-payment-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" } +frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" } frame-system-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -14,6 +17,10 @@ sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-session = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-finality-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -37,7 +44,10 @@ kusama-runtime = { path = "../../runtime/kusama", optional = true } westend-runtime = { path = "../../runtime/westend", optional = true } rococo-runtime = { path = "../../runtime/rococo", optional = true } +polkadot-core-primitives = { path = "../../core-primitives" } polkadot-primitives = { path = "../../primitives" } +polkadot-node-core-parachains-inherent = { path = "../core/parachains-inherent" } +polkadot-runtime-common = { path = "../../runtime/common" } [features] default = ["polkadot"] diff --git a/node/client/src/lib.rs b/node/client/src/lib.rs index e33cd096053b..8446950fe4df 100644 --- a/node/client/src/lib.rs +++ b/node/client/src/lib.rs @@ -24,13 +24,15 @@ use polkadot_primitives::v2::{ }; use sc_client_api::{AuxStore, Backend as BackendT, BlockchainEvents, KeyIterator, UsageProvider}; use sc_executor::NativeElseWasmExecutor; -use sp_api::{CallApiAt, NumberFor, ProvideRuntimeApi}; +use sp_api::{CallApiAt, Encode, NumberFor, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; use sp_consensus::BlockStatus; +use sp_core::Pair; +use sp_keyring::Sr25519Keyring; use sp_runtime::{ generic::{BlockId, SignedBlock}, traits::{BlakeTwo256, Block as BlockT}, - Justifications, + Justifications, OpaqueExtrinsic, }; use sp_storage::{ChildInfo, StorageData, StorageKey}; use std::sync::Arc; @@ -567,3 +569,222 @@ impl sp_blockchain::HeaderBackend for Client { } } } + +/// Provides a `SignedPayload` for any runtime. +/// +/// Should only be used for benchmarking as it is not tested for regular usage. +/// +/// The first code block should set up all variables that are needed to create the +/// `SignedPayload`. The second block can make use of the `SignedPayload`. +/// +/// This is not done as a trait function since the return type depends on the runtime. +/// This macro therefore uses the same approach as [`with_client!`]. +macro_rules! with_signed_payload { + { + $self:ident, + { + $extra:ident, + $client:ident, + $raw_payload:ident + }, + { + $( $setup:tt )* + }, + ( + $period:expr, + $current_block:expr, + $nonce:expr, + $tip:expr, + $call:expr, + $genesis:expr + ), + { + $( $usage:tt )* + } + } => { + match $self { + #[cfg(feature = "polkadot")] + Self::Polkadot($client) => { + use polkadot_runtime as runtime; + + $( $setup )* + + let $extra: runtime::SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), + frame_system::CheckSpecVersion::::new(), + frame_system::CheckTxVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckMortality::::from(sp_runtime::generic::Era::mortal( + $period, + $current_block, + )), + frame_system::CheckNonce::::from($nonce), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from($tip), + polkadot_runtime_common::claims::PrevalidateAttests::::new(), + ); + + let $raw_payload = runtime::SignedPayload::from_raw( + $call.clone(), + $extra.clone(), + ( + (), + runtime::VERSION.spec_version, + runtime::VERSION.transaction_version, + $genesis.clone(), + $genesis, + (), + (), + (), + (), + ), + ); + + $( $usage )* + }, + #[cfg(feature = "westend")] + Self::Westend($client) => { + use westend_runtime as runtime; + + $( $setup )* + + signed_payload!($extra, $raw_payload, + ($period, $current_block, $nonce, $tip, $call, $genesis)); + + $( $usage )* + }, + #[cfg(feature = "kusama")] + Self::Kusama($client) => { + use kusama_runtime as runtime; + + $( $setup )* + + signed_payload!($extra, $raw_payload, + ($period, $current_block, $nonce, $tip, $call, $genesis)); + + $( $usage )* + }, + #[cfg(feature = "rococo")] + Self::Rococo($client) => { + use rococo_runtime as runtime; + + $( $setup )* + + signed_payload!($extra, $raw_payload, + ($period, $current_block, $nonce, $tip, $call, $genesis)); + + $( $usage )* + }, + } + } +} + +/// Generates a `SignedPayload` for the Kusama, Westend and Rococo runtime. +/// +/// Should only be used for benchmarking as it is not tested for regular usage. +#[allow(unused_macros)] +macro_rules! signed_payload { + ( + $extra:ident, $raw_payload:ident, + ( + $period:expr, + $current_block:expr, + $nonce:expr, + $tip:expr, + $call:expr, + $genesis:expr + ) + ) => { + let $extra: runtime::SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), + frame_system::CheckSpecVersion::::new(), + frame_system::CheckTxVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckMortality::::from( + sp_runtime::generic::Era::mortal($period, $current_block), + ), + frame_system::CheckNonce::::from($nonce), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from($tip), + ); + + let $raw_payload = runtime::SignedPayload::from_raw( + $call.clone(), + $extra.clone(), + ( + (), + runtime::VERSION.spec_version, + runtime::VERSION.transaction_version, + $genesis.clone(), + $genesis, + (), + (), + (), + ), + ); + }; +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for Client { + fn remark(&self, nonce: u32) -> std::result::Result { + with_signed_payload! { + self, + {extra, client, raw_payload}, + { + // First the setup code to init all the variables that are needed + // to build the signed extras. + use runtime::{Call, SystemCall}; + + let call = Call::System(SystemCall::remark { remark: vec![] }); + let bob = Sr25519Keyring::Bob.pair(); + + let period = polkadot_runtime_common::BlockHashCount::get().checked_next_power_of_two().map(|c| c / 2).unwrap_or(2) as u64; + + let current_block = 0; + let tip = 0; + let genesis = client.usage_info().chain.best_hash; + }, + (period, current_block, nonce, tip, call, genesis), + /* The SignedPayload is generated here */ + { + // Use the payload to generate a signature. + let signature = raw_payload.using_encoded(|payload| bob.sign(payload)); + + let ext = runtime::UncheckedExtrinsic::new_signed( + call, + sp_runtime::AccountId32::from(bob.public()).into(), + polkadot_core_primitives::Signature::Sr25519(signature.clone()), + extra, + ); + Ok(ext.into()) + } + } + } +} + +/// Generates inherent data for benchmarking Polkadot, Kusama, Westend and Rococo. +/// +/// Not to be used outside of benchmarking since it returns mocked values. +pub fn benchmark_inherent_data( + header: polkadot_core_primitives::Header, +) -> std::result::Result { + use sp_inherents::InherentDataProvider; + let mut inherent_data = sp_inherents::InherentData::new(); + + // Assume that all runtimes have the `timestamp` pallet. + let d = std::time::Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + timestamp.provide_inherent_data(&mut inherent_data)?; + + let para_data = polkadot_primitives::v2::InherentData { + bitfields: Vec::new(), + backed_candidates: Vec::new(), + disputes: Vec::new(), + parent_header: header, + }; + + polkadot_node_core_parachains_inherent::ParachainsInherentDataProvider::from_data(para_data) + .provide_inherent_data(&mut inherent_data)?; + + Ok(inherent_data) +} diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index 1feb7c5ffe3d..e7b465be1744 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -42,6 +42,11 @@ pub struct ParachainsInherentDataProvider { } impl ParachainsInherentDataProvider { + /// Create a [`Self`] directly from some [`ParachainsInherentData`]. + pub fn from_data(inherent_data: ParachainsInherentData) -> Self { + Self { inherent_data } + } + /// Create a new instance of the [`ParachainsInherentDataProvider`]. pub async fn create>( client: &C, diff --git a/runtime/kusama/src/lib.rs b/runtime/kusama/src/lib.rs index adf550ca7e0d..77dbec08f06c 100644 --- a/runtime/kusama/src/lib.rs +++ b/runtime/kusama/src/lib.rs @@ -79,6 +79,7 @@ use sp_version::NativeVersion; use sp_version::RuntimeVersion; use static_assertions::const_assert; +pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_election_provider_multi_phase::Call as EPMCall; #[cfg(feature = "std")] diff --git a/runtime/polkadot/src/lib.rs b/runtime/polkadot/src/lib.rs index 55cdf6373922..691cd7771fb7 100644 --- a/runtime/polkadot/src/lib.rs +++ b/runtime/polkadot/src/lib.rs @@ -81,6 +81,7 @@ use sp_version::NativeVersion; use sp_version::RuntimeVersion; use static_assertions::const_assert; +pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_election_provider_multi_phase::Call as EPMCall; #[cfg(feature = "std")] diff --git a/runtime/rococo/src/lib.rs b/runtime/rococo/src/lib.rs index 19682b45d2d9..904fcbdeaed0 100644 --- a/runtime/rococo/src/lib.rs +++ b/runtime/rococo/src/lib.rs @@ -75,6 +75,8 @@ use bridge_runtime_common::messages::{ source::estimate_message_dispatch_and_delivery_fee, MessageBridge, }; +pub use frame_system::Call as SystemCall; + /// Constant values used within the runtime. use rococo_runtime_constants::{currency::*, fee::*, time::*}; diff --git a/runtime/westend/src/lib.rs b/runtime/westend/src/lib.rs index 01e766023bfb..30f6d4b64d4c 100644 --- a/runtime/westend/src/lib.rs +++ b/runtime/westend/src/lib.rs @@ -74,6 +74,7 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use sp_version::NativeVersion; use sp_version::RuntimeVersion; +pub use frame_system::Call as SystemCall; pub use pallet_election_provider_multi_phase::Call as EPMCall; #[cfg(feature = "std")] pub use pallet_staking::StakerStatus; diff --git a/tests/benchmark_overhead_works.rs b/tests/benchmark_overhead_works.rs new file mode 100644 index 000000000000..19d949a5bfbc --- /dev/null +++ b/tests/benchmark_overhead_works.rs @@ -0,0 +1,69 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::{process::Command, result::Result}; +use tempfile::tempdir; + +static RUNTIMES: [&'static str; 6] = ["polkadot", "kusama", "westend", "rococo", "wococo", "versi"]; + +/// `benchmark-overhead` works for all dev runtimes. +#[test] +fn benchmark_overhead_works() { + for runtime in RUNTIMES { + let runtime = format!("{}-dev", runtime); + assert!(benchmark_overhead(runtime).is_ok()); + } +} + +/// `benchmark-overhead` rejects all non-dev runtimes. +#[test] +fn benchmark_overhead_rejects_non_dev_runtimes() { + for runtime in RUNTIMES { + assert!(benchmark_overhead(runtime.into()).is_err()); + } +} + +fn benchmark_overhead(runtime: String) -> Result<(), String> { + let tmp_dir = tempdir().expect("could not create a temp dir"); + let base_path = tmp_dir.path(); + + // Invoke `benchmark-overhead` with all options to make sure that they are valid. + let status = Command::new(cargo_bin("polkadot")) + .args(["benchmark-overhead", "--chain", &runtime]) + .arg("-d") + .arg(base_path) + .arg("--weight-path") + .arg(base_path) + .args(["--warmup", "5", "--repeat", "5"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + // Only put 5 extrinsics into the block otherwise it takes forever to build it + // especially for a non-release builds. + .args(["--max-ext-per-block", "5"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + if !status.success() { + return Err("Command failed".into()) + } + + // Weight files have been created. + assert!(base_path.join("block_weights.rs").exists()); + assert!(base_path.join("extrinsic_weights.rs").exists()); + Ok(()) +}