Improve online_delete configuration and DB tuning:

* Document delete_batch, back_off_milliseconds, age_threshold_seconds.
* Convert those time values to chrono types.
* Fix bug that ignored age_threshold_seconds.
* Add a "recovery buffer" to the config that gives the node a chance to
  recover before aborting online delete.
* Add begin/end log messages around the SQL queries.
* Add a new configuration section: [sqlite] to allow tuning the sqlite
  database operations. Ignored on full/large history servers.
* Update documentation of [node_db] and [sqlite] in the
  rippled-example.cfg file.
* Resolves XRPLF#3321

cfg/rippled-example.cfg

@@ -36,7 +36,7 @@
 # For more information on where the rippled server instance searches for the
 # file, visit:
 #
-# https://developers.ripple.com/commandline-usage.html#generic-options
+# https://xrpl.org/commandline-usage.html#generic-options
 #
 # This file should be named rippled.cfg. This file is UTF-8 with DOS, UNIX,
 # or Mac style end of lines. Blank lines and lines beginning with '#' are
@@ -869,18 +869,65 @@
 #
 # These keys are possible for any type of backend:
 #
+# earliest_seq The default is 32570 to match the XRP ledger
+# network's earliest allowed sequence. Alternate
+# networks may set this value. Minimum value of 1.
+# If a [shard_db] section is defined, and this
+# value is present either [node_db] or [shard_db],
+# it must be defined with the same value in both
+# sections.
+#
 # online_delete Minimum value of 256. Enable automatic purging
 # of older ledger information. Maintain at least this
 # number of ledger records online. Must be greater
 # than or equal to ledger_history.
 #
-# advisory_delete 0 for disabled, 1 for enabled. If set, then
-# require administrative RPC call "can_delete"
-# to enable online deletion of ledger records.
+# These keys modify the behavior of online_delete, and thus are only
+# relevant if online_delete is defined and non-zero:
 #
-# earliest_seq The default is 32570 to match the XRP ledger
-# network's earliest allowed sequence. Alternate
-# networks may set this value. Minimum value of 1.
+# advisory_delete 0 for disabled, 1 for enabled. If set, the
+# administrative RPC call "can_delete" is required
+# to enable online deletion of ledger records.
+# Online deletion does not run automatically if
+# non-zero and the last deletion was on a ledger
+# greater than the current "can_delete" setting.
+# Default is 0.
+#
+# delete_batch When automatically purging, SQLite database
+# records are deleted in batches. This value
+# controls the maximum size of each batch. Larger
+# batches keep the databases locked for more time,
+# which may cause other functions to fall behind,
+# and thus cause the node to lose sync.
+# Default is 100.
+#
+# back_off_milliseconds
+# Number of milliseconds to wait between
+# online_delete batches to allow other functions
+# to catch up.
+# Default is 100.
+#
+# age_threshold_seconds
+# The online delete process will only run if the
+# latest validated ledger is younger than this
+# number of seconds.
+# Default is 60.
+#
+# recovery_wait_seconds
+# The online delete process checks periodically
+# that rippled is still in sync with the network,
+# and that the validated ledger is less than
+# 'age_threshold_seconds' old. By default, if it
+# is not the online delete process aborts and
+# tries again later. If 'recovery_wait_seconds'
+# is set and rippled is out of sync, but likely to
+# recover quickly, then online delete will wait
+# this number of seconds for rippled to get back
+# into sync before it aborts.
+# Set this value if the node is otherwise staying
+# in sync, or recovering quickly, but the online
+# delete process is unable to finish.
+# Default is unset.
 #
 # Notes:
 # The 'node_db' entry configures the primary, persistent storage.
@@ -892,6 +939,12 @@
 # [import_db] Settings for performing a one-time import (optional)
 # [database_path] Path to the book-keeping databases.
 #
+# The server creates and maintains 4 to 5 bookkeeping SQLite databases in
+# the 'database_path' location. If you omit this configuration setting,
+# the server creates a directory called "db" located in the same place as
+# your rippled.cfg file.
+# Partial pathnames are relative to the location of the rippled executable.
+#
 # [shard_db] Settings for the Shard Database (optional)
 #
 # Format (without spaces):
@@ -907,12 +960,84 @@
 #
 # max_size_gb Maximum disk space the database will utilize (in gigabytes)
 #
+# [sqlite] Tuning settings for the SQLite databases (optional)
+#
+# Format (without spaces):
+# One or more lines of case-insensitive key / value pairs:
+# <key> '=' <value>
+# ...
+#
+# Example 1:
+# sync_level=low
+#
+# Example 2:
+# journal_mode=off
+# synchronous=off
+#
+# WARNING: These settings can have significant effects on data integrity,
+# particularly in systemic failure scenarios. It is strongly recommended
+# that they be left at their defaults unless the server is having
+# performance issues during normal operation or during automatic purging
+# (online_delete) operations. A warning will be logged on startup if
+# 'ledger_history' is configured to store more than 10,000,000 ledgers and
+# any of these settings are less safe than the default. This is due to the
+# inordinate amount of time and bandwidth it will take to safely rebuild a
+# corrupted database of that size from other peers.
+#
+# Optional keys:
 #
-# There are 4 bookkeeping SQLite database that the server creates and
-# maintains. If you omit this configuration setting, it will default to
-# creating a directory called "db" located in the same place as your
-# rippled.cfg file. Partial pathnames will be considered relative to
-# the location of the rippled executable.
+# safety_level Valid values: high, low
+# The default is "high", which tunes the SQLite
+# databases in the most reliable mode, and is
+# equivalent to:
+# journal_mode=wal
+# synchronous=normal
+# temp_store=file
+# "low" is equivalent to:
+# journal_mode=memory
+# synchronous=off
+# temp_store=memory
+# These "low" settings trade speed and reduced I/O
+# for a higher risk of data loss. See the
+# individual settings below for more information.
+# This setting may not be combined with any of the
+# other tuning settings: "journal_mode",
+# "synchronous", or "temp_store".
+#
+# journal_mode Valid values: delete, truncate, persist, memory, wal, off
+# The default is "wal", which uses a write-ahead
+# log to implement database transactions.
+# Alternately, "memory" saves disk I/O, but if
+# rippled crashes during a transaction, the
+# database is likely to be corrupted.
+# See https://www.sqlite.org/pragma.html#pragma_journal_mode
+# for more details about the available options.
+# This setting may not be combined with the
+# "safety_level" setting.
+#
+# synchronous Valid values: off, normal, full, extra
+# The default is "normal", which works well with
+# the "wal" journal mode. Alternatively, "off"
+# allows rippled to continue as soon as data is
+# passed to the OS, which can significantly
+# increase speed, but risks data corruption if
+# the host computer crashes before writing that
+# data to disk.
+# See https://www.sqlite.org/pragma.html#pragma_synchronous
+# for more details about the available options.
+# This setting may not be combined with the
+# "safety_level" setting.
+#
+# temp_store Valid values: default, file, memory
+# The default is "file", which will use files
+# for temporary database tables and indices.
+# Alternatively, "memory" may save I/O, but
+# rippled does not currently use many, if any,
+# of these temporary objects.
+# See https://www.sqlite.org/pragma.html#pragma_temp_store
+# for more details about the available options.
+# This setting may not be combined with the
+# "safety_level" setting.
 #
 #
 #
@@ -1212,24 +1337,27 @@ medium
 
 # This is primary persistent datastore for rippled. This includes transaction
 # metadata, account states, and ledger headers. Helpful information can be
-# found here: https://ripple.com/wiki/NodeBackEnd
-# delete old ledgers while maintaining at least 2000. Do not require an
-# external administrative command to initiate deletion.
+# found at https://xrpl.org/capacity-planning.html#node-db-type
+# type=NuDB is recommended for non-validators with fast SSDs. Validators or
+# slow / spinning disks should use RocksDB. Caution: Spinning disks are
+# not recommended. They do not perform well enough to consistently remain
+# synced to the network.
+# online_delete=512 is recommended to delete old ledgers while maintaining at
+# least 512.
+# advisory_delete=0 allows the online delete process to run automatically
+# when the node has approximately two times the "online_delete" value of
+# ledgers. No external administrative command is required to initiate
+# deletion.
 [node_db]
-type=RocksDB
-path=/var/lib/rippled/db/rocksdb
-open_files=2000
-filter_bits=12
-cache_mb=256
-file_size_mb=8
-file_size_mult=2
-online_delete=2000
+type=NuDB
+path=/var/lib/rippled/db/nudb
+online_delete=512
 advisory_delete=0
 
 # This is the persistent datastore for shards. It is important for the health
 # of the ripple network that rippled operators shard as much as practical.
-# NuDB requires SSD storage. Helpful information can be found here
-# https://ripple.com/build/history-sharding
+# NuDB requires SSD storage. Helpful information can be found at
+# https://xrpl.org/history-sharding.html
 #[shard_db]
 #path=/var/lib/rippled/db/shards/nudb
 #max_size_gb=500
@@ -1248,7 +1376,8 @@ time.apple.com
 time.nist.gov
 pool.ntp.org
 
-# To use the XRP test network (see https://ripple.com/build/xrp-test-net/),
+# To use the XRP test network
+# (see https://xrpl.org/connect-your-rippled-to-the-xrp-test-net.html),
 # use the following [ips] section:
 # [ips]
 # r.altnet.rippletest.net 51235

src/ripple/app/ledger/Ledger.cpp

@@ -228,14 +228,14 @@ Ledger::Ledger(
  !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
  {
  loaded = false;
- JLOG(j.warn()) << "Don't have TX root for ledger";
+ JLOG(j.warn()) << "Don't have transaction root for ledger" << info_.seq;
  }
 
  if (info_.accountHash.isNonZero() &&
  !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
  {
  loaded = false;
- JLOG(j.warn()) << "Don't have AS root for ledger";
+ JLOG(j.warn()) << "Don't have state data root for ledger" << info_.seq;
  }
 
  txMap_->setImmutable();

src/ripple/app/main/Application.cpp

@@ -1019,7 +1019,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
 
  try
  {
- auto const setup = setup_DatabaseCon(*config_);
+ auto setup = setup_DatabaseCon(*config_, m_journal);
 
  // transaction database
  mTxnDB = std::make_unique<DatabaseCon>(
@@ -1069,6 +1069,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
  mLedgerDB->setupCheckpointing(m_jobQueue.get(), logs());
 
  // wallet database
+ setup.useGlobalPragma = false;
  mWalletDB = std::make_unique<DatabaseCon>(
  setup,
  WalletDBName,
@@ -1360,7 +1361,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
  JLOG(m_journal.fatal())
  << "Free SQLite space for transaction db is less than "
  "512MB. To fix this, rippled must be executed with the "
- "vacuum <sqlitetmpdir> parameter before restarting. "
+ "\"--vacuum\" parameter before restarting. "
  "Note that this activity can take multiple days, "
  "depending on database size.";
  signalStop();

src/ripple/app/main/DBInit.h

@@ -26,13 +26,23 @@ namespace ripple {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+// These pragmas are built at startup and applied to all database
+// connections, unless otherwise noted.
+inline constexpr char const* CommonDBPragmaJournal{"PRAGMA journal_mode=%s;"};
+inline constexpr char const* CommonDBPragmaSync{"PRAGMA synchronous=%s;"};
+inline constexpr char const* CommonDBPragmaTemp{"PRAGMA temp_store=%s;"};
+// A warning will be logged if any lower-safety sqlite tuning settings
+// are used and at least this much ledger history is configured. This
+// includes full history nodes. This is because such a large amount of
+// data will be more difficult to recover if a rare failure occurs,
+// which are more likely with some of the other available tuning settings.
+inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
+
 // Ledger database holds ledgers and ledger confirmations
 inline constexpr auto LgrDBName{"ledger.db"};
 
-inline constexpr std::array<char const*, 3> LgrDBPragma{
- {"PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;",
- "PRAGMA journal_size_limit=1582080;"}};
+inline constexpr std::array<char const*, 1> LgrDBPragma{
+ {"PRAGMA journal_size_limit=1582080;"}};
 
 inline constexpr std::array<char const*, 5> LgrDBInit{
  {"BEGIN TRANSACTION;",
@@ -61,22 +71,13 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
 // Transaction database holds transactions and public keys
 inline constexpr auto TxDBName{"transaction.db"};
 
-inline constexpr
-#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
- std::array<char const*, 6>
- TxDBPragma
+inline constexpr std::array TxDBPragma
 {
- {
-#else
- std::array<char const*, 5> TxDBPragma {{
-#endif
- "PRAGMA page_size=4096;", "PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;", "PRAGMA journal_size_limit=1582080;",
- "PRAGMA max_page_count=2147483646;",
+ "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
+ "PRAGMA max_page_count=2147483646;",
 #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
-  "PRAGMA mmap_size=17179869184;"
+ "PRAGMA mmap_size=17179869184;"
 #endif
- }
 };
 
 inline constexpr std::array<char const*, 8> TxDBInit{
@@ -115,10 +116,8 @@ inline constexpr std::array<char const*, 8> TxDBInit{
 // Temporary database used with an incomplete shard that is being acquired
 inline constexpr auto AcquireShardDBName{"acquire.db"};
 
-inline constexpr std::array<char const*, 3> AcquireShardDBPragma{
- {"PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;",
- "PRAGMA journal_size_limit=1582080;"}};
+inline constexpr std::array<char const*, 1> AcquireShardDBPragma{
+ {"PRAGMA journal_size_limit=1582080;"}};
 
 inline constexpr std::array<char const*, 1> AcquireShardDBInit{
  {"CREATE TABLE IF NOT EXISTS Shard ( \
@@ -130,6 +129,7 @@ inline constexpr std::array<char const*, 1> AcquireShardDBInit{
 ////////////////////////////////////////////////////////////////////////////////
 
 // Pragma for Ledger and Transaction databases with complete shards
+// These override the CommonDBPragma values defined above.
 inline constexpr std::array<char const*, 2> CompleteShardDBPragma{
  {"PRAGMA synchronous=OFF;", "PRAGMA journal_mode=OFF;"}};
 
@@ -172,6 +172,7 @@ inline constexpr std::array<char const*, 6> WalletDBInit{
 
 static constexpr auto stateDBName{"state.db"};
 
+// These override the CommonDBPragma values defined above.
 static constexpr std::array<char const*, 2> DownloaderDBPragma{
  {"PRAGMA synchronous=FULL;", "PRAGMA journal_mode=DELETE;"}};