Expose state-db memory info (#5110)

This exposes memory statistics from the state-db.

client/api/src/client.rs

@@ -97,13 +97,56 @@ pub struct ClientInfo<Block: BlockT> {
  pub usage: Option<UsageInfo>,
 }
 
+/// A wrapper to store the size of some memory.
+#[derive(Default, Clone, Debug, Copy)]
+pub struct MemorySize(usize);
+
+impl MemorySize {
+ /// Creates `Self` from the given `bytes` size.
+ pub fn from_bytes(bytes: usize) -> Self {
+ Self(bytes)
+ }
+
+ /// Returns the memory size as bytes.
+ pub fn as_bytes(self) -> usize {
+ self.0
+ }
+}
+
+impl fmt::Display for MemorySize {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ if self.0 < 1024 {
+ write!(f, "{} bytes", self.0)
+ } else if self.0 < 1024 * 1024 {
+ write!(f, "{:.2} KiB", self.0 as f64 / 1024f64)
+ } else if self.0 < 1024 * 1024 * 1024 {
+ write!(f, "{:.2} MiB", self.0 as f64 / (1024f64 * 1024f64))
+ } else {
+ write!(f, "{:.2} GiB", self.0 as f64 / (1024f64 * 1024f64 * 1024f64))
+ }
+ }
+}
+
+/// Memory statistics for state db.
+#[derive(Default, Clone, Debug)]
+pub struct StateDbMemoryInfo {
+ /// Memory usage of the non-canonical overlay
+ pub non_canonical: MemorySize,
+ /// Memory usage of the pruning window.
+ pub pruning: Option<MemorySize>,
+ /// Memory usage of the pinned blocks.
+ pub pinned: MemorySize,
+}
+
 /// Memory statistics for client instance.
 #[derive(Default, Clone, Debug)]
 pub struct MemoryInfo {
  /// Size of state cache.
- pub state_cache: usize,
+ pub state_cache: MemorySize,
  /// Size of backend database cache.
- pub database_cache: usize,
+ pub database_cache: MemorySize,
+ /// Size of the state db.
+ pub state_db: StateDbMemoryInfo,
 }
 
 /// I/O statistics for client instance.
@@ -144,10 +187,16 @@ pub struct UsageInfo {
 
 impl fmt::Display for UsageInfo {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(f,
- "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)",
+ write!(
+ f,
+ "caches: ({} state, {} db overlay), \
+ state db: ({} non-canonical, {} pruning, {} pinned), \
+ i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)",
  self.memory.state_cache,
  self.memory.database_cache,
+ self.memory.state_db.non_canonical,
+ self.memory.state_db.pruning.unwrap_or_default(),
+ self.memory.state_db.pinned,
  self.io.transactions,
  self.io.bytes_written,
  self.io.bytes_read,

client/db/src/lib.rs

@@ -46,9 +46,11 @@ use std::path::PathBuf;
 use std::io;
 use std::collections::HashMap;
 
-use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo};
-use sc_client_api::backend::NewBlockState;
-use sc_client_api::backend::PrunableStateChangesTrieStorage;
+use sc_client_api::{
+ ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo, MemorySize,
+ execution_extensions::ExecutionExtensions,
+ backend::{NewBlockState, PrunableStateChangesTrieStorage},
+};
 use sp_blockchain::{
  Result as ClientResult, Error as ClientError,
  well_known_cache_keys, HeaderBackend,
@@ -1455,13 +1457,17 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
  self.state_usage.take(),
  )
  );
- let database_cache = parity_util_mem::malloc_size(&*self.storage.db);
- let state_cache = (*&self.shared_cache).lock().used_storage_cache_size();
+ let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.storage.db));
+ let state_cache = MemorySize::from_bytes(
+ (*&self.shared_cache).lock().used_storage_cache_size(),
+ );
+ let state_db = self.storage.state_db.memory_info();
 
  Some(UsageInfo {
  memory: MemoryInfo {
  state_cache,
  database_cache,
+ state_db,
  },
  io: IoInfo {
  transactions: io_stats.transactions,

client/db/src/light.rs

@@ -317,7 +317,7 @@ impl<Block: BlockT> LightStorage<Block> {
  // if the header includes changes trie root, let's build a changes tries roots CHT
  if header.digest().log(DigestItem::as_changes_trie_root).is_some() {
  let mut current_num = new_cht_start;
- let cht_range = ::std::iter::from_fn(|| {
+ let cht_range = std::iter::from_fn(|| {
  let old_current_num = current_num;
  current_num = current_num + One::one();
  Some(old_current_num)
@@ -572,15 +572,16 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 
  #[cfg(not(target_os = "unknown"))]
  fn usage_info(&self) -> Option<UsageInfo> {
- use sc_client_api::{MemoryInfo, IoInfo};
+ use sc_client_api::{MemoryInfo, IoInfo, MemorySize};
 
- let database_cache = parity_util_mem::malloc_size(&*self.db);
+ let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.db));
  let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious));
 
  Some(UsageInfo {
  memory: MemoryInfo {
  database_cache,
- state_cache: 0,
+ state_cache: Default::default(),
+ state_db: Default::default(),
  },
  io: IoInfo {
  transactions: io_stats.transactions,

client/informant/src/lib.rs

@@ -46,7 +46,10 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur
  if let Some(ref usage) = info.usage {
  trace!(target: "usage", "Usage statistics: {}", usage);
  } else {
- trace!(target: "usage", "Usage statistics not displayed as backend does not provide it")
+ trace!(
+ target: "usage",
+ "Usage statistics not displayed as backend does not provide it",
+ )
  }
  #[cfg(not(target_os = "unknown"))]
  trace!(

client/service/src/builder.rs

@@ -1085,10 +1085,18 @@ ServiceBuilder<
  "finalized_hash" => ?info.chain.finalized_hash,
  "bandwidth_download" => bandwidth_download,
  "bandwidth_upload" => bandwidth_upload,
- "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0),
- "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0),
- "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0),
- "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0),
+ "used_state_cache_size" => info.usage.as_ref()
+ .map(|usage| usage.memory.state_cache.as_bytes())
+ .unwrap_or(0),
+ "used_db_cache_size" => info.usage.as_ref()
+ .map(|usage| usage.memory.database_cache.as_bytes())
+ .unwrap_or(0),
+ "disk_read_per_sec" => info.usage.as_ref()
+ .map(|usage| usage.io.bytes_read)
+ .unwrap_or(0),
+ "disk_write_per_sec" => info.usage.as_ref()
+ .map(|usage| usage.io.bytes_written)
+ .unwrap_or(0),
  );
  if let Some(metrics) = metrics.as_ref() {
  metrics.memory_usage_bytes.set(memory);

client/state-db/Cargo.toml

@@ -11,8 +11,11 @@ description = "State database maintenance. Handles canonicalization and pruning
 [dependencies]
 parking_lot = "0.10.0"
 log = "0.4.8"
+sc-client-api = { version = "2.0.0-alpha.2", path = "../api" }
 sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" }
 codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+parity-util-mem = "0.5.1"
+parity-util-mem-derive = "0.1.0"
 
 [dev-dependencies]
 env_logger = "0.7.0"