Improve online_delete configuration and DB tuning:

cfg/rippled-example.cfg

@@ -869,18 +869,65 @@
 #
 # These keys are possible for any type of backend:
 #
+# earliest_seq The default is 32570 to match the XRP ledger
+# network's earliest allowed sequence. Alternate
+# networks may set this value. Minimum value of 1.
+# If a [shard_db] section is defined, and this
+# value is present either [node_db] or [shard_db],
+# it must be defined with the same value in both
+# sections.
+#
 # online_delete Minimum value of 256. Enable automatic purging
 # of older ledger information. Maintain at least this
 # number of ledger records online. Must be greater
 # than or equal to ledger_history.
 #
-# advisory_delete 0 for disabled, 1 for enabled. If set, then
-# require administrative RPC call "can_delete"
-# to enable online deletion of ledger records.
+# These keys modify the behavior of online_delete, and thus are only
+# relevant if online_delete is defined and non-zero:
 #
-# earliest_seq The default is 32570 to match the XRP ledger
-# network's earliest allowed sequence. Alternate
-# networks may set this value. Minimum value of 1.
+# advisory_delete 0 for disabled, 1 for enabled. If set, the
+# administrative RPC call "can_delete" is required
+# to enable online deletion of ledger records.
+# Online deletion does not run automatically if
+# non-zero and the last deletion was on a ledger
+# greater than the current "can_delete" setting.
+# Default is 0.
+#
+# delete_batch When automatically purging, SQLite database
+# records are deleted in batches. This value
+# controls the maximum size of each batch. Larger
+# batches keep the databases locked for more time,
+# which may cause other functions to fall behind,
+# and thus cause the node to lose sync.
+# Default is 100.
+#
+# back_off_milliseconds
+# Number of milliseconds to wait between
+# online_delete batches to allow other functions
+# to catch up.
+# Default is 100.
+#
+# age_threshold_seconds
+# The online delete process will only run if the
+# latest validated ledger is younger than this
+# number of seconds.
+# Default is 60.
+#
+# recovery_buffer_seconds
+# The online delete process checks periodically
+# that rippled is still in sync with the network,
+# and that the validated ledger is less than
+# 'age_threshold_seconds' old. By default, if it
+# is not the online delete process aborts and
+# tries again later. If 'recovery_buffer_seconds'
+# is set and rippled is out of sync, but likely to
+# recover quickly, then online delete will wait
+# this number of seconds for rippled to get back
+# into sync before it aborts.
+# Set this value if the node is otherwise staying
+# in sync, or recovering quickly, but the online
+# delete process is unable to finish.
+# Default is unset.
 #
 # Notes:
 # The 'node_db' entry configures the primary, persistent storage.
@@ -892,6 +939,12 @@
 # [import_db] Settings for performing a one-time import (optional)
 # [database_path] Path to the book-keeping databases.
 #
+# There are 4 or 5 bookkeeping SQLite database that the server creates and
+# maintains. If you omit this configuration setting, it will default to
+# creating a directory called "db" located in the same place as your
+# rippled.cfg file. Partial pathnames will be considered relative to
+# the location of the rippled executable.
+#
 # [shard_db] Settings for the Shard Database (optional)
 #
 # Format (without spaces):
@@ -907,12 +960,64 @@
 #
 # max_size_gb Maximum disk space the database will utilize (in gigabytes)
 #
+# [sqlite] Tuning settings for the SQLite databases (optional)
 #
-# There are 4 bookkeeping SQLite database that the server creates and
-# maintains. If you omit this configuration setting, it will default to
-# creating a directory called "db" located in the same place as your
-# rippled.cfg file. Partial pathnames will be considered relative to
-# the location of the rippled executable.
+# Format (without spaces):
+# One or more lines of case-insensitive key / value pairs:
+# <key> '=' <value>
+# ...
+#
+# Example:
+# sync_level=low
+# journal_mode=off
+#
+# WARNING: These settings can have significant effects on data integrity,
+# particularly in failure scenarios. It is strongly recommended that they
+# be left at their defaults unless the server is having performance issues
+# during normal operation or during automatic purging (online_delete)
+# operations.
+#
+# Optional keys:
+#
+# safety_level Valid values: high, low
+# The default is "high", and tunes the SQLite
+# databases in the most reliable mode. "low"
+# is equivalent to
+# journal_mode=memory
+# synchronous=off
+# temp_store=memory
+# These settings trade speed and reduced I/O
+# for a higher risk of data loss. See the
+# individual settings below for more information.
+#
+# journal_mode Valid values: delete, truncate, persist, memory, wal, off
+# The default is "wal", which uses a write-ahead
+# log to implement database transactions.
+# Alternately, "memory" saves disk I/O, but if
+# rippled crashes during a transaction, the
+# database is likely to be corrupted.
+# See https://www.sqlite.org/pragma.html#pragma_journal_mode
+# for more details about the available options.
+#
+# synchronous Valid values: off, normal, full, extra
+# The default is "normal", which works well with
+# the "wal" journal mode. Alternatively, "off"
+# allows rippled to continue as soon as data is
+# passed to the OS, which can significantly
+# increase speed, but risks data corruption if
+# the host computer crashes before writing that
+# data to disk.
+# See https://www.sqlite.org/pragma.html#pragma_synchronous
+# for more details about the available options.
+#
+# temp_store Valid values: default, file, memory
+# The default is "file", which will use files
+# for temporary database tables and indices.
+# Alternatively, "memory" may save I/O, but
+# rippled does not currently use many, if any,
+# of these temporary objects.
+# See https://www.sqlite.org/pragma.html#pragma_temp_store
+# for more details about the available options.
 #
 #
 #
@@ -1212,23 +1317,24 @@ medium
 
 # This is primary persistent datastore for rippled. This includes transaction
 # metadata, account states, and ledger headers. Helpful information can be
-# found here: https://ripple.com/wiki/NodeBackEnd
-# delete old ledgers while maintaining at least 2000. Do not require an
-# external administrative command to initiate deletion.
+# found at https://xrpl.org/capacity-planning.html#node-db-type
+# type=NuDB is recommended for non-validators with fast SSDs. Validators or
+# slow / spinning disks should use RocksDB.
+# online_delete=512 is recommended to delete old ledgers while maintaining at
+# least 512.
+# advisory_delete=0 allows the online delete process to run automatically
+# when the node has approximately two times the "online_delete" value of
+# ledgers. No external administrative command is required to initiate
+# deletion.
 [node_db]
-type=RocksDB
-path=/var/lib/rippled/db/rocksdb
-open_files=2000
-filter_bits=12
-cache_mb=256
-file_size_mb=8
-file_size_mult=2
-online_delete=2000
+type=NuDB
+path=/var/lib/rippled/db/nudb
+online_delete=512
 advisory_delete=0
 
 # This is the persistent datastore for shards. It is important for the health
 # of the ripple network that rippled operators shard as much as practical.
-# NuDB requires SSD storage. Helpful information can be found here
+# NuDB requires SSD storage. Helpful information can be found at
 # https://ripple.com/build/history-sharding
 #[shard_db]
 #path=/var/lib/rippled/db/shards/nudb

src/ripple/app/ledger/Ledger.cpp

@@ -228,14 +228,14 @@ Ledger::Ledger(
  !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
  {
  loaded = false;
- JLOG(j.warn()) << "Don't have TX root for ledger";
+ JLOG(j.warn()) << "Don't have TX root for ledger" << info_.seq;
  }
 
  if (info_.accountHash.isNonZero() &&
  !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
  {
  loaded = false;
- JLOG(j.warn()) << "Don't have AS root for ledger";
+ JLOG(j.warn()) << "Don't have AS root for ledger" << info_.seq;
  }
 
  txMap_->setImmutable();

src/ripple/app/ledger/LedgerMaster.h

@@ -54,6 +54,10 @@ class Transaction;
 class LedgerMaster : public Stoppable, public AbstractFetchPackContainer
 {
 public:
+ // Age for last validated ledger if the process has yet to validate.
+ static constexpr std::chrono::seconds NO_VALIDATED_LEDGER_AGE =
+ std::chrono::hours{24 * 14};
+
  explicit LedgerMaster(
  Application& app,
  Stopwatch& stopwatch,

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -269,7 +269,7 @@ LedgerMaster::getValidatedLedgerAge()
  if (valClose == 0s)
  {
  JLOG(m_journal.debug()) << "No validated ledger";
- return weeks{2};
+ return NO_VALIDATED_LEDGER_AGE;
  }
 
  std::chrono::seconds ret = app_.timeKeeper().closeTime().time_since_epoch();

src/ripple/app/main/Application.cpp

@@ -1026,7 +1026,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
 
  // transaction database
  mTxnDB = std::make_unique<DatabaseCon>(
- setup, TxDBName, TxDBPragma, TxDBInit);
+ setup, TxDBName, true, TxDBPragma, TxDBInit);
  mTxnDB->getSession() << boost::str(
  boost::format("PRAGMA cache_size=-%d;") %
  kilobytes(config_->getValueFor(SizedItem::txnDBCache)));
@@ -1065,7 +1065,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
 
  // ledger database
  mLedgerDB = std::make_unique<DatabaseCon>(
- setup, LgrDBName, LgrDBPragma, LgrDBInit);
+ setup, LgrDBName, true, LgrDBPragma, LgrDBInit);
  mLedgerDB->getSession() << boost::str(
  boost::format("PRAGMA cache_size=-%d;") %
  kilobytes(config_->getValueFor(SizedItem::lgrDBCache)));
@@ -1075,6 +1075,7 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp
  mWalletDB = std::make_unique<DatabaseCon>(
  setup,
  WalletDBName,
+ false,
  std::array<char const*, 0>(),
  WalletDBInit);
  }

src/ripple/app/main/DBInit.h

@@ -26,13 +26,23 @@ namespace ripple {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+// These pragmas are built at startup and applied to all database
+// connections, unless otherwise noted.
+inline constexpr char const* CommonDBPragmaJournal{"PRAGMA journal_mode=%s;"};
+inline constexpr char const* CommonDBPragmaSync{"PRAGMA synchronous=%s;"};
+inline constexpr char const* CommonDBPragmaTemp{"PRAGMA temp_store=%s;"};
+// Default values will always be used for the common pragmas if
+// at least this much ledger history is configured. This includes
+// full history nodes. This is because such a large amount of data will
+// be more difficult to recover if a rare failure occurs, which are
+// more likely with some of the other available tuning settings.
+inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 100'000'000;
+
 // Ledger database holds ledgers and ledger confirmations
 inline constexpr auto LgrDBName{"ledger.db"};
 
-inline constexpr std::array<char const*, 3> LgrDBPragma{
- {"PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;",
- "PRAGMA journal_size_limit=1582080;"}};
+inline constexpr std::array<char const*, 1> LgrDBPragma{
+ {"PRAGMA journal_size_limit=1582080;"}};
 
 inline constexpr std::array<char const*, 5> LgrDBInit{
  {"BEGIN TRANSACTION;",
@@ -63,15 +73,14 @@ inline constexpr auto TxDBName{"transaction.db"};
 
 inline constexpr
 #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
- std::array<char const*, 6>
+ std::array<char const*, 4>
  TxDBPragma
 {
  {
 #else
- std::array<char const*, 5> TxDBPragma {{
+ std::array<char const*, 3> TxDBPragma {{
 #endif
- "PRAGMA page_size=4096;", "PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;", "PRAGMA journal_size_limit=1582080;",
+ "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
  "PRAGMA max_page_count=2147483646;",
 #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
  "PRAGMA mmap_size=17179869184;"
@@ -115,10 +124,8 @@ inline constexpr std::array<char const*, 8> TxDBInit{
 // Temporary database used with an incomplete shard that is being acquired
 inline constexpr auto AcquireShardDBName{"acquire.db"};
 
-inline constexpr std::array<char const*, 3> AcquireShardDBPragma{
- {"PRAGMA synchronous=NORMAL;",
- "PRAGMA journal_mode=WAL;",
- "PRAGMA journal_size_limit=1582080;"}};
+inline constexpr std::array<char const*, 1> AcquireShardDBPragma{
+ {"PRAGMA journal_size_limit=1582080;"}};
 
 inline constexpr std::array<char const*, 1> AcquireShardDBInit{
  {"CREATE TABLE IF NOT EXISTS Shard ( \
@@ -130,6 +137,7 @@ inline constexpr std::array<char const*, 1> AcquireShardDBInit{
 ////////////////////////////////////////////////////////////////////////////////
 
 // Pragma for Ledger and Transaction databases with complete shards
+// These override the CommonDBPragma values defined above.
 inline constexpr std::array<char const*, 2> CompleteShardDBPragma{
  {"PRAGMA synchronous=OFF;", "PRAGMA journal_mode=OFF;"}};
 
@@ -172,6 +180,7 @@ inline constexpr std::array<char const*, 6> WalletDBInit{
 
 static constexpr auto stateDBName{"state.db"};
 
+// These override the CommonDBPragma values defined above.
 static constexpr std::array<char const*, 2> DownloaderDBPragma{
  {"PRAGMA synchronous=FULL;", "PRAGMA journal_mode=DELETE;"}};
 

src/ripple/app/main/Main.cpp

@@ -542,7 +542,7 @@ run(int argc, char** argv)
  }
 
  auto txnDB = std::make_unique<DatabaseCon>(
- dbSetup, TxDBName, TxDBPragma, TxDBInit);
+ dbSetup, TxDBName, false, TxDBPragma, TxDBInit);
  auto& session = txnDB->getSession();
  std::uint32_t pageSize;
 
@@ -555,7 +555,9 @@ run(int argc, char** argv)
  session << "PRAGMA temp_store_directory=\"" << tmpPath.string()
  << "\";";
  session << "VACUUM;";
- session << "PRAGMA journal_mode=WAL;";
+ assert(dbSetup.CommonPragma);
+ for (auto const& p : *dbSetup.CommonPragma)
+ session << p;
  session << "PRAGMA page_size;", soci::into(pageSize);
 
  std::cout << "VACUUM finished. page_size: " << pageSize

src/ripple/app/misc/NetworkOPs.cpp

@@ -2757,16 +2757,24 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
  if (std::abs(closeOffset.count()) >= 60)
  l[jss::close_time_offset] = closeOffset.count();
 
- auto lCloseTime = lpClosed->info().closeTime;
- auto closeTime = app_.timeKeeper().closeTime();
- if (lCloseTime <= closeTime)
+ constexpr std::chrono::seconds HIGH_AGE_THRESHOLD{1000000};
+ if (m_ledgerMaster.haveValidated())
  {
- using namespace std::chrono_literals;
- auto age = closeTime - lCloseTime;
- if (age < 1000000s)
- l[jss::age] = Json::UInt(age.count());
- else
- l[jss::age] = 0;
+ auto const age = m_ledgerMaster.getValidatedLedgerAge();
+ l[jss::age] =
+ Json::UInt(age < HIGH_AGE_THRESHOLD ? age.count() : 0);
+ }
+ else
+ {
+ auto lCloseTime = lpClosed->info().closeTime;
+ auto closeTime = app_.timeKeeper().closeTime();
+ if (lCloseTime <= closeTime)
+ {
+ using namespace std::chrono_literals;
+ auto age = closeTime - lCloseTime;
+ l[jss::age] =
+ Json::UInt(age < HIGH_AGE_THRESHOLD ? age.count() : 0);
+ }
  }
  }