Merge pull request #78 from turbofish-org/upgrade

Upgrade dependencies, `Send` + `Sync`

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  override: true
  - name: Cache
  uses: actions/cache@v3
@@ -45,7 +45,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  override: true
  - name: Cache
  uses: actions/cache@v3
@@ -72,7 +72,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  components: llvm-tools-preview
  override: true
  - name: Cache
@@ -111,7 +111,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  components: rustfmt
  override: true
  - name: Check
@@ -128,7 +128,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  components: clippy
  override: true
  - name: Cache
@@ -156,7 +156,7 @@ jobs:
  - name: Use Nightly
  uses: actions-rs/toolchain@v1
  with:
- toolchain: nightly-2022-11-18
+ toolchain: nightly-2024-04-25
  override: true
  - name: Cache
  uses: actions/cache@v3

Cargo.toml

@@ -7,32 +7,16 @@ edition = "2018"
 license = "MIT"
 
 [dependencies]
-thiserror= "1.0.31"
-sha2 = "0.10.2"
-log = "0.4.17"
-
-[dependencies.time]
-version = "0.3.11"
-optional = true
-
-[dependencies.hex]
-version = "0.4.3"
-optional = true
+thiserror= "1.0.58"
+sha2 = "0.10.8"
+log = "0.4.21"
 
 [dependencies.colored]
-version = "2.0.0"
+version = "2.1.0"
 optional = true
 
 [dependencies.num_cpus]
-version = "1.13.1"
-optional = true
-
-[dependencies.byteorder]
-version = "1.4.3"
-optional = true
-
-[dependencies.failure]
-version = "0.1.8"
+version = "1.16.0"
 optional = true
 
 [dependencies.ed]
@@ -45,16 +29,16 @@ features = ["small_rng"]
 optional = true
 
 [dependencies.rocksdb]
-version = "0.21.0"
+version = "0.22.0"
 default-features = false
 optional = true
 
 [dependencies.jemallocator]
-version = "0.5.0"
+version = "0.5.4"
 features = ["disable_initial_exec_tls"]
 optional = true
 
 [features]
 default = ["full", "verify"]
-full = ["rand", "rocksdb", "time", "hex", "colored", "num_cpus", "byteorder", "failure", "ed"]
-verify = ["ed", "failure"]
+full = ["rand", "rocksdb", "colored", "num_cpus", "ed"]
+verify = ["ed"]

rustfmt.toml

@@ -0,0 +1,3 @@
+comment_width = 80
+wrap_comments = true
+

src/lib.rs

@@ -12,7 +12,8 @@ mod error;
 /// The top-level store API.
 #[cfg(feature = "full")]
 mod merk;
-/// Provides a container type that allows temporarily taking ownership of a value.
+/// Provides a container type that allows temporarily taking ownership of a
+/// value.
 // TODO: move this into its own crate
 pub mod owner;
 /// Algorithms for generating and verifying Merkle proofs.

src/merk/chunks.rs

@@ -9,8 +9,10 @@ use ed::Encode;
 use rocksdb::DBRawIterator;
 
 /// A `ChunkProducer` allows the creation of chunk proofs, used for trustlessly
-/// replicating entire Merk trees. Chunks can be generated on the fly in a
-/// random order, or iterated in order for slightly better performance.
+/// replicating entire Merk trees.
+///
+/// Chunks can be generated on the fly in a random order, or iterated in order
+/// for slightly better performance.
 pub struct ChunkProducer<'a> {
  trunk: Vec<Op>,
  chunk_boundaries: Vec<Vec<u8>>,
@@ -51,8 +53,8 @@ impl<'a> ChunkProducer<'a> {
  }
 
  /// Gets the chunk with the given index. Errors if the index is out of
- /// bounds or the tree is empty - the number of chunks can be checked by calling
- /// `producer.len()`.
+ /// bounds or the tree is empty - the number of chunks can be checked by
+ /// calling `producer.len()`.
  pub fn chunk(&mut self, index: usize) -> Result<Vec<u8>> {
  if index >= self.len() {
  return Err(Error::IndexOutOfBounds("Chunk index out-of-bounds".into()));

src/merk/mod.rs

@@ -2,10 +2,10 @@ pub mod chunks;
 pub mod restore;
 pub mod snapshot;
 
-use std::cell::Cell;
 use std::cmp::Ordering;
 use std::collections::LinkedList;
 use std::path::{Path, PathBuf};
+use std::sync::RwLock;
 
 use rocksdb::DB;
 use rocksdb::{checkpoint::Checkpoint, ColumnFamilyDescriptor, WriteBatch};
@@ -30,7 +30,7 @@ fn column_families() -> Vec<ColumnFamilyDescriptor> {
 
 /// A handle to a Merkle key/value store backed by RocksDB.
 pub struct Merk {
- pub(crate) tree: Cell<Option<Tree>>,
+ pub(crate) tree: RwLock<Option<Tree>>,
  pub(crate) db: rocksdb::DB,
  pub(crate) path: PathBuf,
 }
@@ -58,7 +58,7 @@ impl Merk {
  )?;
 
  Ok(Merk {
- tree: Cell::new(load_root(&db)?),
+ tree: RwLock::new(load_root(&db)?),
  db,
  path: path_buf,
  })
@@ -75,7 +75,7 @@ impl Merk {
  let db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;
 
  Ok(Merk {
- tree: Cell::new(load_root(&db)?),
+ tree: RwLock::new(load_root(&db)?),
  db,
  path: path_buf,
  })
@@ -124,7 +124,7 @@ impl Merk {
  /// proofs can be checked against). If the tree is empty, returns the null
  /// hash (zero-filled).
  pub fn root_hash(&self) -> Hash {
- self.use_tree(|maybe_tree| root_hash(maybe_tree))
+ self.use_tree(root_hash)
  }
 
  /// Applies a batch of operations (puts and deletes) to the tree.
@@ -190,14 +190,12 @@ impl Merk {
  /// unsafe { store.apply_unchecked(batch, &[]).unwrap() };
  /// ```
  pub unsafe fn apply_unchecked(&mut self, batch: &Batch, aux: &Batch) -> Result<()> {
- let maybe_walker = self
- .tree
- .take()
- .take()
- .map(|tree| Walker::new(tree, self.source()));
+ let mut tree = self.tree.write().unwrap();
+ let maybe_walker = tree.take().map(|tree| Walker::new(tree, self.source()));
 
  let (maybe_tree, deleted_keys) = Walker::apply_to(maybe_walker, batch, self.source())?;
- self.tree.set(maybe_tree);
+ *tree = maybe_tree;
+ drop(tree);
 
  // commit changes to db
  self.commit(deleted_keys, aux)
@@ -300,9 +298,7 @@ impl Merk {
  Q: Into<QueryItem>,
  I: IntoIterator<Item = Q>,
  {
- self.use_tree_mut(move |maybe_tree| {
- prove_unchecked(maybe_tree, self.source(), query.into_iter())
- })
+ self.use_tree_mut(move |maybe_tree| prove_unchecked(maybe_tree, self.source(), query))
  }
 
  pub fn flush(&self) -> Result<()> {
@@ -360,13 +356,11 @@ impl Merk {
  }
 
  pub fn walk<T>(&self, f: impl FnOnce(Option<RefWalker<MerkSource>>) -> T) -> T {
- let mut tree = self.tree.take();
+ let mut tree = self.tree.write().unwrap();
  let maybe_walker = tree
  .as_mut()
  .map(|tree| RefWalker::new(tree, self.source()));
- let res = f(maybe_walker);
- self.tree.set(tree);
- res
+ f(maybe_walker)
  }
 
  pub fn raw_iter(&self) -> rocksdb::DBRawIterator {
@@ -391,16 +385,15 @@ impl Merk {
  }
 
  fn use_tree<T>(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T {
- let tree = self.tree.take();
- let res = f(tree.as_ref());
- self.tree.set(tree);
- res
+ let tree = self.tree.read().unwrap();
+ f(tree.as_ref())
  }
 
  fn use_tree_mut<T>(&self, f: impl FnOnce(Option<&mut Tree>) -> T) -> T {
- let mut tree = self.tree.take();
+ let mut tree_slot = self.tree.write().unwrap();
+ let mut tree = tree_slot.take();
  let res = f(tree.as_mut());
- self.tree.set(tree);
+ *tree_slot = tree;
  res
  }
 
@@ -425,7 +418,7 @@ impl Merk {
 
  pub(crate) fn load_root(&mut self) -> Result<()> {
  let root = load_root(&self.db)?;
- self.tree = Cell::new(root);
+ *self.tree.write().unwrap() = root;
  Ok(())
  }
 }
@@ -680,7 +673,7 @@ mod test {
  let mut merk = Merk::open(&path).unwrap();
  let batch = make_batch_seq(1..10_000);
  merk.apply(batch.as_slice(), &[]).unwrap();
- let mut tree = merk.tree.take().unwrap();
+ let mut tree = merk.tree.write().unwrap().take().unwrap();
  let walker = RefWalker::new(&mut tree, merk.source());
 
  let mut nodes = vec![];
@@ -689,7 +682,7 @@ mod test {
  };
 
  let merk = TempMerk::open(&path).unwrap();
- let mut tree = merk.tree.take().unwrap();
+ let mut tree = merk.tree.write().unwrap().take().unwrap();
  let walker = RefWalker::new(&mut tree, merk.source());
 
  let mut reopen_nodes = vec![];

src/merk/restore.rs

@@ -14,7 +14,7 @@ use crate::{
 };
 use rocksdb::WriteBatch;
 use std::iter::Peekable;
-use std::{path::Path, u8};
+use std::path::Path;
 
 /// A `Restorer` handles decoding, verifying, and storing chunk proofs to
 /// replicate an entire Merk tree. It expects the chunks to be processed in

src/merk/snapshot.rs

@@ -52,7 +52,7 @@ impl<'a> Snapshot<'a> {
  I: IntoIterator<Item = Q>,
  {
  self.use_tree_mut(move |maybe_tree| {
- super::prove_unchecked(maybe_tree, self.source(), query.into_iter())
+ super::prove_unchecked(maybe_tree, self.source(), query)
  })
  }
 
@@ -120,7 +120,19 @@ struct RocksDBSnapshot<'a> {
  inner: *const (),
 }
 
+// We need this because we have a raw pointer to a RocksDB snapshot, but we
+// know that our usage of it is thread-safe:
+// https:/facebook/rocksdb/blob/main/include/rocksdb/snapshot.h#L15-L16
+unsafe impl Send for StaticSnapshot {}
+unsafe impl Sync for StaticSnapshot {}
+
 impl StaticSnapshot {
+ /// Creates a new static snapshot from a RocksDB snapshot.
+ ///
+ /// # Safety
+ /// This function is unsafe because the `StaticSnapshot` will hold a raw
+ /// pointer to the RocksDB snapshot, and it is the caller's responsibility
+ /// to ensure that the snapshot outlives the `StaticSnapshot`.
  pub unsafe fn with_db<'a>(&self, db: &'a rocksdb::DB) -> Snapshot<'a> {
  let db_ss = RocksDBSnapshot {
  _db: db,
@@ -135,7 +147,13 @@ impl StaticSnapshot {
  }
  }
 
- pub unsafe fn drop<'a>(mut self, db: &'a rocksdb::DB) {
+ /// Drops the snapshot without cleaning up the underlying RocksDB snapshot.
+ ///
+ /// # Safety
+ /// This function is unsafe because it allows to drop the snapshot without
+ /// cleaning up the underlying RocksDB snapshot. Ensure that the snapshot
+ /// is manually freed when this function is called.
+ pub unsafe fn drop(mut self, db: &rocksdb::DB) {
  let mut ss = self.with_db(db);
  ss.should_drop_ss = true;
  self.should_drop = true;
@@ -169,3 +187,25 @@ impl Clone for StaticSnapshot {
  }
  }
 }
+
+#[cfg(test)]
+mod tests {
+ use std::mem::transmute;
+
+ use super::RocksDBSnapshot;
+ use crate::test_utils::TempMerk;
+
+ #[test]
+ fn rocksdb_snapshot_struct_format() {
+ assert_eq!(std::mem::size_of::<rocksdb::Snapshot>(), 16);
+
+ let merk = TempMerk::new().unwrap();
+ let exptected_db_ptr = merk.db() as *const _;
+
+ let ss = merk.db().snapshot();
+ let ss: RocksDBSnapshot = unsafe { transmute(ss) };
+ let db_ptr = ss._db as *const _;
+
+ assert_eq!(exptected_db_ptr, db_ptr);
+ }
+}

src/proofs/chunk.rs

@@ -11,8 +11,10 @@ use crate::error::{Error, Result};
 use crate::tree::{Fetch, RefWalker};
 
 /// The minimum number of layers the trunk will be guaranteed to have before
-/// splitting into multiple chunks. If the tree's height is less than double
-/// this value, the trunk should be verified as a leaf chunk.
+/// splitting into multiple chunks.
+///
+/// If the tree's height is less than double this value, the trunk should be
+/// verified as a leaf chunk.
 pub const MIN_TRUNK_HEIGHT: usize = 5;
 
 impl<'a, S> RefWalker<'a, S>
@@ -415,9 +417,8 @@ mod tests {
  let batch = make_batch_seq(0..31);
  merk.apply(batch.as_slice(), &[]).unwrap();
 
- let root_node = merk.tree.take();
+ let root_node = merk.tree.read().unwrap();
  let root_key = root_node.as_ref().unwrap().key().to_vec();
- merk.tree.set(root_node);
 
  // whole tree as 1 leaf
  let mut iter = merk.db.raw_iterator();