diff --git a/.gitignore b/.gitignore index 87d9b3b8e00..e5952e0de1b 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ pkg CMakeUserPresets.json bld.rippled/ .vscode + +# Suggested in-tree build directory +/.build/ diff --git a/API-CHANGELOG.md b/API-CHANGELOG.md index ae988beeb33..989a1f8056a 100644 --- a/API-CHANGELOG.md +++ b/API-CHANGELOG.md @@ -1,22 +1,16 @@ # API Changelog -This changelog is intended to list all updates to the API. +This changelog is intended to list all updates to the [public API methods](https://xrpl.org/public-api-methods.html). -For info about how API versioning works, view the [XLS-22d spec](https://github.com/XRPLF/XRPL-Standards/discussions/54). For details about the implementation of API versioning, view the [implementation PR](https://github.com/XRPLF/rippled/pull/3155). +For info about how [API versioning](https://xrpl.org/request-formatting.html#api-versioning) works, including examples, please view the [XLS-22d spec](https://github.com/XRPLF/XRPL-Standards/discussions/54). For details about the implementation of API versioning, view the [implementation PR](https://github.com/XRPLF/rippled/pull/3155). API versioning ensures existing integrations and users continue to receive existing behavior, while those that request a higher API version will experience new behavior. The API version controls the API behavior you see. This includes what properties you see in responses, what parameters you're permitted to send in requests, and so on. You specify the API version in each of your requests. When a breaking change is introduced to the `rippled` API, a new version is released. To avoid breaking your code, you should set (or increase) your version when you're ready to upgrade. -For a log of breaking changes, see the **API Version [number]** headings. Breaking changes are associated with a particular API Version number. For non-breaking changes, scroll to the **XRP Ledger version [x.y.z]** headings. Non-breaking changes are associated with a particular XRP Ledger (`rippled`) release. - -## API Version 2 -This version will be supported by `rippled` version 1.12. - -#### V2 account_info response - -`signer_lists` is returned in the root of the response, instead of being nested under `account_data` (as it was in API version 1). ([#3770](https://github.com/XRPLF/rippled/pull/3770)) +For a log of breaking changes, see the **API Version [number]** headings. In general, breaking changes are associated with a particular API Version number. For non-breaking changes, scroll to the **XRP Ledger version [x.y.z]** headings. Non-breaking changes are associated with a particular XRP Ledger (`rippled`) release. ## API Version 1 -This version is supported by `rippled` version 1.11. + +This version is supported by all `rippled` versions. At time of writing, it is the default API version, used when no `api_version` is specified. When a new API version is introduced, the command line interface will default to the latest API version. The command line is intended for ad-hoc usage by humans, not programs or automated scripts. The command line is not meant for use in production code. ### Idiosyncrasies @@ -30,7 +24,7 @@ The `network_id` field was added in the `server_info` response in version 1.5.0 ## XRP Ledger version 1.12.0 -Version 1.12.0 is in development. +[Version 1.12.0](https://github.com/XRPLF/rippled/releases/tag/1.12.0) was released on Sep 6, 2023. ### Additions in 1.12 @@ -76,9 +70,11 @@ Additions are intended to be non-breaking (because they are purely additive). ### Breaking changes in 1.11 -- Added the ability to mark amendments as obsolete. For the `feature` admin API, there is a new possible value for the `vetoed` field. ([#4291](https://github.com/XRPLF/rippled/pull/4291)) -- The API now won't accept seeds or public keys in place of account addresses. ([#4404](https://github.com/XRPLF/rippled/pull/4404)) -- For the `ledger_data` method, when all entries are filtered out, the API now returns an empty list (an empty array, `[]`). (Previously, it would return `null`.) While this is technically a breaking change, the new behavior is consistent with the documentation, so this is considered only a bug fix. ([#4398](https://github.com/XRPLF/rippled/pull/4398)) +- Added the ability to mark amendments as obsolete. For the `feature` admin API, there is a new possible value for the `vetoed` field. (https://github.com/XRPLF/rippled/pull/4291) + - The value of `vetoed` can now be `true`, `false`, or `"Obsolete"`. +- Removed the acceptance of seeds or public keys in place of account addresses. (https://github.com/XRPLF/rippled/pull/4404) + - This simplifies the API and encourages better security practices (i.e. seeds should never be sent over the network). +- For the `ledger_data` method, when all entries are filtered out, the `state` field of the response is now an empty list (in other words, an empty array, `[]`). (Previously, it would return `null`.) While this is technically a breaking change, the new behavior is consistent with the documentation, so this is considered only a bug fix. (https://github.com/XRPLF/rippled/pull/4398) - If and when the `fixNFTokenRemint` amendment activates, there will be a new AccountRoot field, `FirstNFTSequence`. This field is set to the current account sequence when the account issues their first NFT. If an account has not issued any NFTs, then the field is not set. ([#4406](https://github.com/XRPLF/rippled/pull/4406)) - There is a new account deletion restriction: an account can only be deleted if `FirstNFTSequence` + `MintedNFTokens` + `256` is less than the current ledger sequence. - This is potentially a breaking change if clients have logic for determining whether an account can be deleted. @@ -97,3 +93,51 @@ Additions are intended to be non-breaking (because they are purely additive). - Added an `account_flags` object to the `account_info` method response. (https://github.com/XRPLF/rippled/pull/4459) - Added `NFTokenPages` to the `account_objects` RPC. (https://github.com/XRPLF/rippled/pull/4352) - Fixed: `marker` returned from the `account_lines` command would not work on subsequent commands. (https://github.com/XRPLF/rippled/pull/4361) + +# In development + +Changes below this point are in development. + +## API Version 2 + +At the time of writing, this version is expected to be introduced in `rippled` version 2.0. + +Currently (prior to the release of 2.0), it is available as a "beta" version, meaning it can be enabled with a config setting in `rippled.cfg`: +``` +[beta_rpc_api] +1 +``` + +Since `api_version` 2 is in "beta", breaking changes can be made at any time. + +#### Modifications to account_info response in V2 + +- `signer_lists` is returned in the root of the response. Previously, in API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770) +- When using an invalid `signer_lists` value, the API now returns an "invalidParams" error. (https://github.com/XRPLF/rippled/pull/4585) + - (`signer_lists` must be a boolean. In API version 1, strings are accepted and may return a normal response - as if `signer_lists` were `true`.) + +#### Modifications to [account_tx](https://xrpl.org/account_tx.html#account_tx) response in V2 + +- Using `ledger_index_min`, `ledger_index_max`, and `ledger_index` returns `invalidParams` because if you use `ledger_index_min` or `ledger_index_max`, then it does not make sense to also specify `ledger_index`. Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4571) + - The same applies for `ledger_index_min`, `ledger_index_max`, and `ledger_hash`. (https://github.com/XRPLF/rippled/issues/4545#issuecomment-1565065579) +- Using a `ledger_index_min` or `ledger_index_max` beyond the range of ledgers that the server has: + - returns `lgrIdxMalformed` in API version 2. (https://github.com/XRPLF/rippled/issues/4288) + - Previously, in API version 1, no error was returned. + +##### In progress + +- Attempting to use a non-boolean value (such as a string) for the `binary` or `forward` parameters returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620) + +#### Modifications to [noripple_check](https://xrpl.org/noripple_check.html#noripple_check) response in V2 + +##### In progress + +- Attempting to use a non-boolean value (such as a string) for the `transactions` parameter returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620) + +# Unit tests for API changes + +The following information is useful to developers contributing to this project: + +The purpose of unit tests is to catch bugs and prevent regressions. In general, it often makes sense to create a test function when there is a breaking change to the API. For APIs that have changed in a new API version, the tests should be modified so that both the prior version and the new version are properly tested. + +To take one example: for `account_info` version 1, WebSocket and JSON-RPC behavior should be tested. The latest API version, i.e. API version 2, should be tested over WebSocket, JSON-RPC, and command line. diff --git a/BUILD.md b/BUILD.md index cc7e6f9318f..d3078fb39d3 100644 --- a/BUILD.md +++ b/BUILD.md @@ -216,7 +216,7 @@ It patches their CMake to correctly import its dependencies. and make sure it matches the `build_type` setting you chose in the previous step. - Multi-config gnerators: + Multi-config generators: ``` cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake .. diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 95c5e411631..8d2ff6cbaef 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -568,9 +568,7 @@ target_sources (rippled PRIVATE src/ripple/core/impl/JobQueue.cpp src/ripple/core/impl/LoadEvent.cpp src/ripple/core/impl/LoadMonitor.cpp - src/ripple/core/impl/SNTPClock.cpp src/ripple/core/impl/SociDB.cpp - src/ripple/core/impl/TimeKeeper.cpp src/ripple/core/impl/Workers.cpp src/ripple/core/Pg.cpp #[===============================[ @@ -636,6 +634,7 @@ target_sources (rippled PRIVATE src/ripple/overlay/impl/Cluster.cpp src/ripple/overlay/impl/ConnectAttempt.cpp src/ripple/overlay/impl/Handshake.cpp + src/ripple/overlay/impl/InboundHandoff.cpp src/ripple/overlay/impl/Message.cpp src/ripple/overlay/impl/OverlayImpl.cpp src/ripple/overlay/impl/PeerImp.cpp @@ -928,7 +927,6 @@ if (tests) src/test/jtx/impl/AMMTest.cpp src/test/jtx/impl/Env.cpp src/test/jtx/impl/JSONRPCClient.cpp - src/test/jtx/impl/ManualTimeKeeper.cpp src/test/jtx/impl/TestHelpers.cpp src/test/jtx/impl/WSClient.cpp src/test/jtx/impl/acctdelete.cpp diff --git a/Builds/CMake/deps/Boost.cmake b/Builds/CMake/deps/Boost.cmake index 23ea5e549cc..041c2380e12 100644 --- a/Builds/CMake/deps/Boost.cmake +++ b/Builds/CMake/deps/Boost.cmake @@ -1,4 +1,4 @@ -find_package(Boost 1.70 REQUIRED +find_package(Boost 1.82 REQUIRED COMPONENTS chrono container @@ -6,6 +6,7 @@ find_package(Boost 1.70 REQUIRED coroutine date_time filesystem + json program_options regex system @@ -29,6 +30,7 @@ target_link_libraries(ripple_boost Boost::coroutine Boost::date_time Boost::filesystem + Boost::json Boost::program_options Boost::regex Boost::system diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index 7a4f4404321..832b548e5de 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -173,7 +173,6 @@ test.nodestore > test.unit_test test.overlay > ripple.app test.overlay > ripple.basics test.overlay > ripple.beast -test.overlay > ripple.core test.overlay > ripple.overlay test.overlay > ripple.peerfinder test.overlay > ripple.protocol diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 2300d476c45..5a40c10cd49 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -30,69 +30,63 @@ If you operate an XRP Ledger server, upgrade to version 1.12.0 by September 20, On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html). +The XRPL Foundation publishes portable binaries, which are drop-in replacements for the `rippled` daemon. [See information and downloads for the portable binaries](https://github.com/XRPLF/rippled-portable-builds#portable-builds-of-the-rippled-server). This will work on most distributions, including Ubuntu 16.04, 18.04, 20.04, and 22.04; CentOS; and others. Please test and open issues on GitHub if there are problems. + ## Changelog -### New Features and Amendments +### Amendments, New Features, and Changes +(These are changes which may impact or be useful to end users. For example, you may be able to update your code/workflow to take advantage of these changes.) - **`AMM`**: Introduces an automated market maker (AMM) protocol to the XRP Ledger's decentralized exchange, enabling you to trade assets without a counterparty. For more information about AMMs, see: [Automated Market Maker](https://opensource.ripple.com/docs/xls-30d-amm/amm-uc/). [#4294](https://github.com/XRPLF/rippled/pull/4294) -- **`Clawback`**: Allows issuers to add the `lsfAllowTrustLineClawback` flag to an issuing account. This enables the account to recover, or _claw back_, issued tokens after they're distributed to accounts. For additional documentation on this feature, see: [#4553](https://github.com/XRPLF/rippled/pull/4553). +- **`Clawback`**: Adds a setting, *Allow Clawback*, which lets an issuer recover, or _claw back_, tokens that they previously issued. Issuers cannot enable this setting if they have issued tokens already. For additional documentation on this feature, see: [#4553](https://github.com/XRPLF/rippled/pull/4553). - **`fixReducedOffersV1`**: Reduces the occurrence of order books that are blocked by reduced offers. [#4512](https://github.com/XRPLF/rippled/pull/4512) -- Added binary hardening and linker flags to enhance security during the build process. [#4603](https://github.com/XRPLF/rippled/pull/4603) - -- Updated build dependencies to the most recent versions in Conan Center. [#4595](https://github.com/XRPLF/rippled/pull/4595) - -- Updated Conan recipe for NuDB. [#4615](https://github.com/XRPLF/rippled/pull/4615) - -- Added a pre-commit hook that runs the clang-format linter locally before committing changes. To install this feature, see: [CONTRIBUTING](https://github.com/XRPLF/xrpl-dev-portal/blob/master/CONTRIBUTING.md). [#4599](https://github.com/XRPLF/rippled/pull/4599) - -- Added quality-of-life improvements to workflows, using new [concurrency control](https://docs.github.com/en/actions/using-jobs/using-concurrency) features. [#4597](https://github.com/XRPLF/rippled/pull/4597) - -- Added an Artifactory to the `nix` workflow to improve build times. [#4556](https://github.com/XRPLF/rippled/pull/4556) - - Added WebSocket and RPC port info to `server_info` responses. [#4427](https://github.com/XRPLF/rippled/pull/4427) +- Removed the deprecated `accepted`, `seqNum`, `hash`, and `totalCoins` fields from the `ledger` method. [#4244](https://github.com/XRPLF/rippled/pull/4244) ### Bug Fixes and Performance Improvements +(These are behind-the-scenes improvements, such as internal changes to the code, which are not expected to impact end users.) -- Fixed an incorrect error response when there are missing fields in the API v2 `ledger_entry` method. [#4552](https://github.com/XRPLF/rippled/pull/4552) - -- Updated checkout versions to resolve warnings during Github jobs. [#4598](https://github.com/XRPLF/rippled/pull/4598) +- Added a pre-commit hook that runs the clang-format linter locally before committing changes. To install this feature, see: [CONTRIBUTING](https://github.com/XRPLF/xrpl-dev-portal/blob/master/CONTRIBUTING.md). [#4599](https://github.com/XRPLF/rippled/pull/4599) -- Added an error response to the API v2 `account_info` method when you include an invalid `signer_lists` value. [#4585](https://github.com/XRPLF/rippled/pull/4585) +- In order to make it more straightforward to catch and handle overflows: changed the output type of the `mulDiv()` function from `std::pair` to `std::optional`. [#4243](https://github.com/XRPLF/rippled/pull/4243) -- Fixed an issue with the debug package build. [#4591](https://github.com/XRPLF/rippled/pull/4591) +- Updated `Handler::Condition` enum values to make the code less brittle. [#4239](https://github.com/XRPLF/rippled/pull/4239) -- Added additional error responses to the API v2 `AccountTx` method. [#4571](https://github.com/XRPLF/rippled/pull/4571) +- Renamed `ServerHandlerImp` to `ServerHandler`. [#4516](https://github.com/XRPLF/rippled/pull/4516), [#4592](https://github.com/XRPLF/rippled/pull/4592) -- Fixed build references to deleted `ServerHandlerImp`. [#4592](https://github.com/XRPLF/rippled/pull/4592) +- Replaced hand-rolled code with `std::from_chars` for better maintainability. [#4473](https://github.com/XRPLF/rippled/pull/4473) -- Fixed package definitions for Conan. [#4485](https://github.com/XRPLF/rippled/pull/4485) +- Removed an unused `TypedField` move constructor. [#4567](https://github.com/XRPLF/rippled/pull/4567) -- Changed the output type of the `mulDiv()` function from `std::pair` to `std::optional`. [#4243](https://github.com/XRPLF/rippled/pull/4243) -- Updated `Handler::Condition` enum values to make the code less brittle. [#4239](https://github.com/XRPLF/rippled/pull/4239) +### Docs and Build System -- Renamed `ServerHandlerImp` to `ServerHandler`. [#4516](https://github.com/XRPLF/rippled/pull/4516) +- Updated checkout versions to resolve warnings during GitHub jobs. [#4598](https://github.com/XRPLF/rippled/pull/4598) -- Removed the deprecated `accepted`, `seq`, `hash`, and `totalCoins` fields from the `ledger` method. [#4244](https://github.com/XRPLF/rippled/pull/4244) +- Fixed an issue with the Debian package build. [#4591](https://github.com/XRPLF/rippled/pull/4591) -- Replaced hand-rolled code with `std::from_chars` for better maintainability. [#4473](https://github.com/XRPLF/rippled/pull/4473) +- Updated build instructions with additional steps to take after updating dependencies. [#4623](https://github.com/XRPLF/rippled/pull/4623) -- Removed an unused `TypedField` move constructor. [#4567](https://github.com/XRPLF/rippled/pull/4567) +- Updated contributing doc to clarify that beta releases should also be pushed to the `release` branch. [#4589](https://github.com/XRPLF/rippled/pull/4589) -- Enabled the `BETA_RPC_API` flag in the default unit tests config, making the API v2 available to all unit tests. [#4573](https://github.com/XRPLF/rippled/pull/4573) +- Enabled the `BETA_RPC_API` flag in the default unit tests config, making the API v2 (beta) available to unit tests. [#4573](https://github.com/XRPLF/rippled/pull/4573) +- Conan dependency management. + - Fixed package definitions for Conan. [#4485](https://github.com/XRPLF/rippled/pull/4485) + - Updated build dependencies to the most recent versions in Conan Center. [#4595](https://github.com/XRPLF/rippled/pull/4595) + - Updated Conan recipe for NuDB. [#4615](https://github.com/XRPLF/rippled/pull/4615) -### Docs +- Added binary hardening and linker flags to enhance security during the build process. [#4603](https://github.com/XRPLF/rippled/pull/4603) -- Updated build instructions with additional steps to take after updating dependencies. [#4623](https://github.com/XRPLF/rippled/pull/4623) +- Added an Artifactory to the `nix` workflow to improve build times. [#4556](https://github.com/XRPLF/rippled/pull/4556) -- Updated contributing doc to clarify that beta releases should also be pushed to the `release` branch. [#4589](https://github.com/XRPLF/rippled/pull/4589) +- Added quality-of-life improvements to workflows, using new [concurrency control](https://docs.github.com/en/actions/using-jobs/using-concurrency) features. [#4597](https://github.com/XRPLF/rippled/pull/4597) [Full Commit Log](https://github.com/XRPLF/rippled/compare/1.11.0...1.12.0) diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 6e1d553a970..e21197eed5a 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -463,19 +463,6 @@ # # # -# [sntp_servers] -# -# IP address or domain of NTP servers to use for time synchronization. -# -# These NTP servers are suitable for rippled servers located in the United -# States: -# time.windows.com -# time.apple.com -# time.nist.gov -# pool.ntp.org -# -# -# # [max_transactions] # # Configure the maximum number of transactions to have in the job queue @@ -1704,12 +1691,6 @@ advisory_delete=0 [debug_logfile] /var/log/rippled/debug.log -[sntp_servers] -time.windows.com -time.apple.com -time.nist.gov -pool.ntp.org - # To use the XRP test network # (see https://xrpl.org/connect-your-rippled-to-the-xrp-test-net.html), # use the following [ips] section: diff --git a/cfg/rippled-reporting.cfg b/cfg/rippled-reporting.cfg index f09c17ae637..632a8a7800e 100644 --- a/cfg/rippled-reporting.cfg +++ b/cfg/rippled-reporting.cfg @@ -450,19 +450,6 @@ # # # -# [sntp_servers] -# -# IP address or domain of NTP servers to use for time synchronization. -# -# These NTP servers are suitable for rippled servers located in the United -# States: -# time.windows.com -# time.apple.com -# time.nist.gov -# pool.ntp.org -# -# -# # [max_transactions] # # Configure the maximum number of transactions to have in the job queue @@ -1662,12 +1649,6 @@ advisory_delete=0 [debug_logfile] /var/log/rippled-reporting/debug.log -[sntp_servers] -time.windows.com -time.apple.com -time.nist.gov -pool.ntp.org - # To use the XRP test network # (see https://xrpl.org/connect-your-rippled-to-the-xrp-test-net.html), # use the following [ips] section: diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 14b816e4564..8fcbb8a971c 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -1175,9 +1175,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) // Optionally turn off logging to console. logs_->silent(config_->silent()); - if (!config_->standalone()) - timeKeeper_->run(config_->SNTP_SERVERS); - if (!initRelationalDatabase() || !initNodeStore()) return false; diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 80ac40132d5..a998640c01c 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -803,10 +803,8 @@ run(int argc, char** argv) if (vm.count("debug")) setDebugLogSink(logs->makeSink("Debug", beast::severities::kTrace)); - auto timeKeeper = make_TimeKeeper(logs->journal("TimeKeeper")); - auto app = make_Application( - std::move(config), std::move(logs), std::move(timeKeeper)); + std::move(config), std::move(logs), std::make_unique()); if (!app->setup(vm)) return -1; diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 4e91a9d32f6..cd7fe3861ec 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -739,11 +739,10 @@ class NetworkOPsImp final : public NetworkOPs sPeerStatus, // Peer status changes. sConsensusPhase, // Consensus phase sBookChanges, // Per-ledger order book changes - - sLastEntry = sBookChanges // as this name implies, any new entry - // must be ADDED ABOVE this one + sLastEntry // Any new entry must be ADDED ABOVE this one }; - std::array mStreamMaps; + + std::array mStreamMaps; ServerFeeSummary mLastFeeSummary; @@ -2614,13 +2613,10 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) lpClosed->fees().accountReserve(0).decimalXRP(); l[jss::reserve_inc_xrp] = lpClosed->fees().increment.decimalXRP(); - auto const nowOffset = app_.timeKeeper().nowOffset(); - if (std::abs(nowOffset.count()) >= 60) - l[jss::system_time_offset] = nowOffset.count(); - - auto const closeOffset = app_.timeKeeper().closeOffset(); - if (std::abs(closeOffset.count()) >= 60) - l[jss::close_time_offset] = closeOffset.count(); + if (auto const closeOffset = app_.timeKeeper().closeOffset(); + std::abs(closeOffset.count()) >= 60) + l[jss::close_time_offset] = + static_cast(closeOffset.count()); #if RIPPLED_REPORTING std::int64_t const dbAge = diff --git a/src/ripple/beast/clock/abstract_clock.h b/src/ripple/beast/clock/abstract_clock.h index e543d6d7bb4..128ab82b4bd 100644 --- a/src/ripple/beast/clock/abstract_clock.h +++ b/src/ripple/beast/clock/abstract_clock.h @@ -70,7 +70,7 @@ class abstract_clock abstract_clock(abstract_clock const&) = default; /** Returns the current time. */ - virtual time_point + [[nodiscard]] virtual time_point now() const = 0; }; diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index 48bc9681e46..a9acd4c6b2b 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -149,9 +149,11 @@ class Config : public BasicConfig bool nodeToShard = false; bool ELB_SUPPORT = false; - std::vector IPS; // Peer IPs from rippled.cfg. - std::vector IPS_FIXED; // Fixed Peer IPs from rippled.cfg. - std::vector SNTP_SERVERS; // SNTP servers from rippled.cfg. + // Entries from [ips] config stanza + std::vector IPS; + + // Entries from [ips_fixed] config stanza + std::vector IPS_FIXED; enum StartUpType { FRESH, NORMAL, LOAD, LOAD_FILE, REPLAY, NETWORK }; StartUpType START_UP = NORMAL; diff --git a/src/ripple/core/TimeKeeper.h b/src/ripple/core/TimeKeeper.h index ebc6c1f1ab0..55970ec8227 100644 --- a/src/ripple/core/TimeKeeper.h +++ b/src/ripple/core/TimeKeeper.h @@ -22,73 +22,99 @@ #include #include -#include -#include -#include +#include namespace ripple { /** Manages various times used by the server. */ class TimeKeeper : public beast::abstract_clock { -public: - virtual ~TimeKeeper() = default; - - /** Launch the internal thread. +private: + std::atomic closeOffset_{}; - The internal thread synchronizes local network time - using the provided list of SNTP servers. - */ - virtual void - run(std::vector const& servers) = 0; - - /** Returns the estimate of wall time, in network time. + // Adjust system_clock::time_point for NetClock epoch + static constexpr time_point + adjust(std::chrono::system_clock::time_point when) + { + return time_point(std::chrono::duration_cast( + when.time_since_epoch() - days(10957))); + } - The network time is wall time adjusted for the Ripple - epoch, the beginning of January 1st, 2000 UTC. Each server - can compute a different value for network time. Other - servers value for network time is not directly observable, - but good guesses can be made by looking at validators' - positions on close times. +public: + virtual ~TimeKeeper() = default; - Servers compute network time by adjusting a local wall - clock using SNTP and then adjusting for the epoch. - */ - virtual time_point - now() const override = 0; + /** Returns the current time, using the server's clock. - /** Returns the close time, in network time. + It's possible for servers to have a different value for network + time, especially if they do not use some external mechanism for + time synchronization (e.g. NTP or SNTP). This is fine. - Close time is the time the network would agree that - a ledger closed, if a ledger closed right now. + This estimate is not directly visible to other servers over the + protocol, but it is possible for them to make an educated guess + if this server publishes proposals or validations. - The close time represents the notional "center" - of the network. Each server assumes its clock - is correct, and tries to pull the close time towards - its measure of network time. + @note The network time is adjusted for the "Ripple epoch" which + was arbitrarily defined as 2000-01-01T00:00:00Z by Arthur + Britto and David Schwartz during early development of the + code. No rationale has been provided for this curious and + annoying, but otherwise unimportant, choice. */ - virtual time_point - closeTime() const = 0; + [[nodiscard]] time_point + now() const override + { + return adjust(std::chrono::system_clock::now()); + } - /** Adjust the close time. + /** Returns the predicted close time, in network time. - This is called in response to received validations. + The predicted close time represents the notional "center" of the + network. Each server assumes that its clock is correct and tries + to pull the close time towards its measure of network time. */ - virtual void - adjustCloseTime(std::chrono::duration amount) = 0; + [[nodiscard]] time_point + closeTime() const + { + return now() + closeOffset_.load(); + } // This may return a negative value - virtual std::chrono::duration - nowOffset() const = 0; - - // This may return a negative value - virtual std::chrono::duration - closeOffset() const = 0; + [[nodiscard]] std::chrono::seconds + closeOffset() const + { + return closeOffset_.load(); + } + + /** Adjust the close time, based on the network's view of time. */ + std::chrono::seconds + adjustCloseTime(std::chrono::seconds by) + { + using namespace std::chrono_literals; + + auto offset = closeOffset_.load(); + + if (by == 0s && offset == 0s) + return offset; + + // The close time adjustment is serialized externally to this + // code. The compare/exchange only serves as a weak check and + // should not fail. Even if it does, it's safe to simply just + // skip the adjustment. + closeOffset_.compare_exchange_strong(offset, [by, offset]() { + // Ignore small offsets and push the close time + // towards our wall time. + if (by > 1s) + return offset + ((by + 3s) / 4); + + if (by < -1s) + return offset + ((by - 3s) / 4); + + return (offset * 3) / 4; + }()); + + return closeOffset_.load(); + } }; -extern std::unique_ptr -make_TimeKeeper(beast::Journal j); - } // namespace ripple #endif diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index f835ca8df04..2f26b8ba525 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -206,14 +206,10 @@ parseIniFile(std::string const& strInput, const bool bTrim) IniFileSections::mapped_type* getIniFileSection(IniFileSections& secSource, std::string const& strSection) { - IniFileSections::iterator it; - IniFileSections::mapped_type* smtResult; - it = secSource.find(strSection); - if (it == secSource.end()) - smtResult = nullptr; - else - smtResult = &(it->second); - return smtResult; + if (auto it = secSource.find(strSection); it != secSource.end()) + return &(it->second); + + return nullptr; } bool @@ -223,22 +219,21 @@ getSingleSection( std::string& strValue, beast::Journal j) { - IniFileSections::mapped_type* pmtEntries = - getIniFileSection(secSource, strSection); - bool bSingle = pmtEntries && 1 == pmtEntries->size(); + auto const pmtEntries = getIniFileSection(secSource, strSection); - if (bSingle) + if (pmtEntries && pmtEntries->size() == 1) { strValue = (*pmtEntries)[0]; + return true; } - else if (pmtEntries) + + if (pmtEntries) { - JLOG(j.warn()) << boost::str( - boost::format("Section [%s]: requires 1 line not %d lines.") % - strSection % pmtEntries->size()); + JLOG(j.warn()) << "Section '" << strSection << "': requires 1 line not " + << pmtEntries->size() << " lines."; } - return bSingle; + return false; } //------------------------------------------------------------------------------ @@ -461,9 +456,6 @@ Config::loadFromString(std::string const& fileContents) if (auto s = getIniFileSection(secConfig, SECTION_IPS_FIXED)) IPS_FIXED = *s; - if (auto s = getIniFileSection(secConfig, SECTION_SNTP)) - SNTP_SERVERS = *s; - // if the user has specified ip:port then replace : with a space. { auto replaceColons = [](std::vector& strVec) { diff --git a/src/ripple/core/impl/SNTPClock.cpp b/src/ripple/core/impl/SNTPClock.cpp deleted file mode 100644 index 8651dcbe5a4..00000000000 --- a/src/ripple/core/impl/SNTPClock.cpp +++ /dev/null @@ -1,491 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -static uint8_t SNTPQueryData[48] = { - 0x1B, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -using namespace std::chrono_literals; -// NTP query frequency - 4 minutes -auto constexpr NTP_QUERY_FREQUENCY = 4min; - -// NTP minimum interval to query same servers - 3 minutes -auto constexpr NTP_MIN_QUERY = 3min; - -// NTP sample window (should be odd) -#define NTP_SAMPLE_WINDOW 9 - -// NTP timestamp constant -auto constexpr NTP_UNIX_OFFSET = 0x83AA7E80s; - -// NTP timestamp validity -auto constexpr NTP_TIMESTAMP_VALID = (NTP_QUERY_FREQUENCY + NTP_MIN_QUERY) * 2; - -// SNTP packet offsets -#define NTP_OFF_INFO 0 -#define NTP_OFF_ROOTDELAY 1 -#define NTP_OFF_ROOTDISP 2 -#define NTP_OFF_REFERENCEID 3 -#define NTP_OFF_REFTS_INT 4 -#define NTP_OFF_REFTS_FRAC 5 -#define NTP_OFF_ORGTS_INT 6 -#define NTP_OFF_ORGTS_FRAC 7 -#define NTP_OFF_RECVTS_INT 8 -#define NTP_OFF_RECVTS_FRAC 9 -#define NTP_OFF_XMITTS_INT 10 -#define NTP_OFF_XMITTS_FRAC 11 - -class SNTPClientImp : public SNTPClock -{ -private: - template - using sys_time = std::chrono::time_point; - - using sys_seconds = sys_time; - - struct Query - { - bool replied; - sys_seconds sent; - std::uint32_t nonce; - - explicit Query(sys_seconds j = sys_seconds::max()) - : replied(false), sent(j) - { - } - }; - - beast::Journal const j_; - std::mutex mutable mutex_; - std::thread thread_; - boost::asio::io_service io_service_; - std::optional work_; - - std::map queries_; - boost::asio::ip::udp::socket socket_; - boost::asio::basic_waitable_timer timer_; - boost::asio::ip::udp::resolver resolver_; - std::vector> servers_; - std::chrono::seconds offset_; - sys_seconds lastUpdate_; - std::deque offsets_; - std::vector buf_; - boost::asio::ip::udp::endpoint ep_; - -public: - using error_code = boost::system::error_code; - - explicit SNTPClientImp(beast::Journal j) - : j_(j) - , work_(io_service_) - , socket_(io_service_) - , timer_(io_service_) - , resolver_(io_service_) - , offset_(0) - , lastUpdate_(sys_seconds::max()) - , buf_(256) - { - } - - ~SNTPClientImp() override - { - if (thread_.joinable()) - { - error_code ec; - timer_.cancel(ec); - socket_.cancel(ec); - work_ = std::nullopt; - thread_.join(); - } - } - - //-------------------------------------------------------------------------- - - void - run(const std::vector& servers) override - { - std::vector::const_iterator it = servers.begin(); - - if (it == servers.end()) - { - JLOG(j_.info()) << "SNTP: no server specified"; - return; - } - - { - std::lock_guard lock(mutex_); - for (auto const& item : servers) - servers_.emplace_back(item, sys_seconds::max()); - } - queryAll(); - - using namespace boost::asio; - socket_.open(ip::udp::v4()); - socket_.bind(ep_); - socket_.async_receive_from( - buffer(buf_, 256), - ep_, - std::bind( - &SNTPClientImp::onRead, - this, - std::placeholders::_1, - std::placeholders::_2)); - timer_.expires_from_now(NTP_QUERY_FREQUENCY); - timer_.async_wait( - std::bind(&SNTPClientImp::onTimer, this, std::placeholders::_1)); - - thread_ = std::thread(&SNTPClientImp::doRun, this); - } - - time_point - now() const override - { - std::lock_guard lock(mutex_); - using namespace std::chrono; - auto const when = time_point_cast(clock_type::now()); - if ((lastUpdate_ == sys_seconds::max()) || - ((lastUpdate_ + NTP_TIMESTAMP_VALID) < - time_point_cast(clock_type::now()))) - return when; - return when + offset_; - } - - duration - offset() const override - { - std::lock_guard lock(mutex_); - return offset_; - } - - //-------------------------------------------------------------------------- - - void - doRun() - { - beast::setCurrentThreadName("rippled: SNTPClock"); - io_service_.run(); - } - - void - onTimer(error_code const& ec) - { - using namespace boost::asio; - if (ec == error::operation_aborted) - return; - if (ec) - { - JLOG(j_.error()) << "SNTPClock::onTimer: " << ec.message(); - return; - } - - doQuery(); - timer_.expires_from_now(NTP_QUERY_FREQUENCY); - timer_.async_wait( - std::bind(&SNTPClientImp::onTimer, this, std::placeholders::_1)); - } - - void - onRead(error_code const& ec, std::size_t bytes_xferd) - { - using namespace boost::asio; - using namespace std::chrono; - if (ec == error::operation_aborted) - return; - - // VFALCO Should we return on any error? - /* - if (ec) - return; - */ - - if (!ec) - { - JLOG(j_.trace()) << "SNTP: Packet from " << ep_; - std::lock_guard lock(mutex_); - auto const query = queries_.find(ep_); - if (query == queries_.end()) - { - JLOG(j_.debug()) << "SNTP: Reply from " << ep_ - << " found without matching query"; - } - else if (query->second.replied) - { - JLOG(j_.debug()) << "SNTP: Duplicate response from " << ep_; - } - else - { - query->second.replied = true; - - if (time_point_cast(clock_type::now()) > - (query->second.sent + 1s)) - { - JLOG(j_.warn()) << "SNTP: Late response from " << ep_; - } - else if (bytes_xferd < 48) - { - JLOG(j_.warn()) << "SNTP: Short reply from " << ep_ << " (" - << bytes_xferd << ") " << buf_.size(); - } - else if ( - reinterpret_cast( - &buf_[0])[NTP_OFF_ORGTS_FRAC] != query->second.nonce) - { - JLOG(j_.warn()) - << "SNTP: Reply from " << ep_ << "had wrong nonce"; - } - else - { - processReply(); - } - } - } - - socket_.async_receive_from( - buffer(buf_, 256), - ep_, - std::bind( - &SNTPClientImp::onRead, - this, - std::placeholders::_1, - std::placeholders::_2)); - } - - //-------------------------------------------------------------------------- - - void - addServer(std::string const& server) - { - std::lock_guard lock(mutex_); - servers_.push_back(std::make_pair(server, sys_seconds::max())); - } - - void - queryAll() - { - while (doQuery()) - { - } - } - - bool - doQuery() - { - std::lock_guard lock(mutex_); - auto best = servers_.end(); - - for (auto iter = servers_.begin(), end = best; iter != end; ++iter) - if ((best == end) || (iter->second == sys_seconds::max()) || - (iter->second < best->second)) - best = iter; - - if (best == servers_.end()) - { - JLOG(j_.trace()) << "SNTP: No server to query"; - return false; - } - - using namespace std::chrono; - auto now = time_point_cast(clock_type::now()); - - if ((best->second != sys_seconds::max()) && - ((best->second + NTP_MIN_QUERY) >= now)) - { - JLOG(j_.trace()) << "SNTP: All servers recently queried"; - return false; - } - - best->second = now; - - boost::asio::ip::udp::resolver::query query( - boost::asio::ip::udp::v4(), best->first, "ntp"); - resolver_.async_resolve( - query, - std::bind( - &SNTPClientImp::resolveComplete, - this, - std::placeholders::_1, - std::placeholders::_2)); - JLOG(j_.trace()) << "SNTPClock: Resolve pending for " << best->first; - return true; - } - - void - resolveComplete( - error_code const& ec, - boost::asio::ip::udp::resolver::iterator it) - { - using namespace boost::asio; - if (ec == error::operation_aborted) - return; - if (ec) - { - JLOG(j_.trace()) << "SNTPClock::resolveComplete: " << ec.message(); - return; - } - - assert(it != ip::udp::resolver::iterator()); - - auto sel = it; - int i = 1; - - while (++it != ip::udp::resolver::iterator()) - { - if (rand_int(i++) == 0) - sel = it; - } - - if (sel != ip::udp::resolver::iterator()) - { - std::lock_guard lock(mutex_); - Query& query = queries_[*sel]; - using namespace std::chrono; - auto now = time_point_cast(clock_type::now()); - - if ((query.sent == now) || ((query.sent + 1s) == now)) - { - // This can happen if the same IP address is reached through - // multiple names - JLOG(j_.trace()) << "SNTP: Redundant query suppressed"; - return; - } - - query.replied = false; - query.sent = now; - query.nonce = rand_int(); - // The following line of code will overflow at 2036-02-07 06:28:16 - // UTC - // due to the 32 bit cast. - reinterpret_cast( - SNTPQueryData)[NTP_OFF_XMITTS_INT] = - static_cast( - (time_point_cast(clock_type::now()) + - NTP_UNIX_OFFSET) - .time_since_epoch() - .count()); - reinterpret_cast( - SNTPQueryData)[NTP_OFF_XMITTS_FRAC] = query.nonce; - socket_.async_send_to( - buffer(SNTPQueryData, 48), - *sel, - std::bind( - &SNTPClientImp::onSend, - this, - std::placeholders::_1, - std::placeholders::_2)); - } - } - - void - onSend(error_code const& ec, std::size_t) - { - if (ec == boost::asio::error::operation_aborted) - return; - - if (ec) - { - JLOG(j_.warn()) << "SNTPClock::onSend: " << ec.message(); - return; - } - } - - void - processReply() - { - using namespace std::chrono; - assert(buf_.size() >= 48); - std::uint32_t* recvBuffer = - reinterpret_cast(&buf_.front()); - - unsigned info = ntohl(recvBuffer[NTP_OFF_INFO]); - auto timev = seconds{ntohl(recvBuffer[NTP_OFF_RECVTS_INT])}; - unsigned stratum = (info >> 16) & 0xff; - - if ((info >> 30) == 3) - { - JLOG(j_.info()) << "SNTP: Alarm condition " << ep_; - return; - } - - if ((stratum == 0) || (stratum > 14)) - { - JLOG(j_.info()) << "SNTP: Unreasonable stratum (" << stratum - << ") from " << ep_; - return; - } - - using namespace std::chrono; - auto now = time_point_cast(clock_type::now()); - timev -= now.time_since_epoch(); - timev -= NTP_UNIX_OFFSET; - - // add offset to list, replacing oldest one if appropriate - offsets_.push_back(timev); - - if (offsets_.size() >= NTP_SAMPLE_WINDOW) - offsets_.pop_front(); - - lastUpdate_ = now; - - // select median time - auto offsetList = offsets_; - std::sort(offsetList.begin(), offsetList.end()); - auto j = offsetList.size(); - auto it = std::next(offsetList.begin(), j / 2); - offset_ = *it; - - if ((j % 2) == 0) - offset_ = (offset_ + (*--it)) / 2; - - // debounce: small corrections likely - // do more harm than good - if ((offset_ == -1s) || (offset_ == 1s)) - offset_ = 0s; - - if (timev != 0s || offset_ != 0s) - { - JLOG(j_.trace()) << "SNTP: Offset is " << timev.count() - << ", new system offset is " << offset_.count(); - } - } -}; - -//------------------------------------------------------------------------------ - -std::unique_ptr -make_SNTPClock(beast::Journal j) -{ - return std::make_unique(j); -} - -} // namespace ripple diff --git a/src/ripple/core/impl/SNTPClock.h b/src/ripple/core/impl/SNTPClock.h deleted file mode 100644 index b63cac53da5..00000000000 --- a/src/ripple/core/impl/SNTPClock.h +++ /dev/null @@ -1,47 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NET_SNTPCLOCK_H_INCLUDED -#define RIPPLE_NET_SNTPCLOCK_H_INCLUDED - -#include -#include -#include -#include -#include -#include - -namespace ripple { - -/** A clock based on system_clock and adjusted for SNTP. */ -class SNTPClock : public beast::abstract_clock -{ -public: - virtual void - run(std::vector const& servers) = 0; - - virtual duration - offset() const = 0; -}; - -extern std::unique_ptr make_SNTPClock(beast::Journal); - -} // namespace ripple - -#endif diff --git a/src/ripple/core/impl/TimeKeeper.cpp b/src/ripple/core/impl/TimeKeeper.cpp deleted file mode 100644 index d1b07f53f44..00000000000 --- a/src/ripple/core/impl/TimeKeeper.cpp +++ /dev/null @@ -1,124 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -namespace ripple { - -class TimeKeeperImpl : public TimeKeeper -{ -private: - beast::Journal const j_; - std::mutex mutable mutex_; - std::chrono::duration closeOffset_; - std::unique_ptr clock_; - - // Adjust system_clock::time_point for NetClock epoch - static time_point - adjust(std::chrono::system_clock::time_point when) - { - return time_point(std::chrono::duration_cast( - when.time_since_epoch() - days(10957))); - } - -public: - explicit TimeKeeperImpl(beast::Journal j) - : j_(j), closeOffset_{}, clock_(make_SNTPClock(j)) - { - } - - void - run(std::vector const& servers) override - { - clock_->run(servers); - } - - time_point - now() const override - { - std::lock_guard lock(mutex_); - return adjust(clock_->now()); - } - - time_point - closeTime() const override - { - std::lock_guard lock(mutex_); - return adjust(clock_->now()) + closeOffset_; - } - - void - adjustCloseTime(std::chrono::duration amount) override - { - using namespace std::chrono; - auto const s = amount.count(); - std::lock_guard lock(mutex_); - // Take large offsets, ignore small offsets, - // push the close time towards our wall time. - if (s > 1) - closeOffset_ += seconds((s + 3) / 4); - else if (s < -1) - closeOffset_ += seconds((s - 3) / 4); - else - closeOffset_ = (closeOffset_ * 3) / 4; - if (closeOffset_.count() != 0) - { - if (std::abs(closeOffset_.count()) < 60) - { - JLOG(j_.info()) << "TimeKeeper: Close time offset now " - << closeOffset_.count(); - } - else - { - JLOG(j_.warn()) << "TimeKeeper: Large close time offset = " - << closeOffset_.count(); - } - } - } - - std::chrono::duration - nowOffset() const override - { - using namespace std::chrono; - using namespace std; - lock_guard lock(mutex_); - return duration_cast>(clock_->offset()); - } - - std::chrono::duration - closeOffset() const override - { - std::lock_guard lock(mutex_); - return closeOffset_; - } -}; - -//------------------------------------------------------------------------------ - -std::unique_ptr -make_TimeKeeper(beast::Journal j) -{ - return std::make_unique(j); -} - -} // namespace ripple diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index df758646540..1242fdc2a68 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -121,7 +121,13 @@ class RPCParser static Json::Value jvParseCurrencyIssuer(std::string const& strCurrencyIssuer) { - static boost::regex reCurIss("\\`([[:alpha:]]{3})(?:/(.+))?\\'"); + // Matches a sequence of 3 characters from + // `ripple::detail::isoCharSet` (the currency), + // optionally followed by a forward slash and some other characters + // (the issuer). + // https://www.boost.org/doc/libs/1_82_0/libs/regex/doc/html/boost_regex/syntax/perl_syntax.html + static boost::regex reCurIss( + "\\`([][:alnum:]<>(){}[|?!@#$%^&*]{3})(?:/(.+))?\\'"); boost::smatch smMatch; diff --git a/src/ripple/overlay/Message.h b/src/ripple/overlay/Message.h index 0d6479366e8..6cb6900c639 100644 --- a/src/ripple/overlay/Message.h +++ b/src/ripple/overlay/Message.h @@ -100,6 +100,14 @@ class Message : public std::enable_shared_from_this return validatorKey_; } + /** Get the message type from the payload header. + * First four bytes are the compression/algorithm flag and the payload size. + * Next two bytes are the message type + * @return Message type + */ + int + getType() const; + private: std::vector buffer_; std::vector bufferCompressed_; @@ -129,15 +137,6 @@ class Message : public std::enable_shared_from_this */ void compress(); - - /** Get the message type from the payload header. - * First four bytes are the compression/algorithm flag and the payload size. - * Next two bytes are the message type - * @param in Payload header pointer - * @return Message type - */ - int - getType(std::uint8_t const* in) const; }; } // namespace ripple diff --git a/src/ripple/overlay/Peer.h b/src/ripple/overlay/Peer.h index ba415974151..dbe5416e590 100644 --- a/src/ripple/overlay/Peer.h +++ b/src/ripple/overlay/Peer.h @@ -39,6 +39,7 @@ enum class ProtocolFeature { ValidatorListPropagation, ValidatorList2Propagation, LedgerReplay, + StartProtocol }; /** Represents a peer connection in the overlay. */ diff --git a/src/ripple/overlay/README.md b/src/ripple/overlay/README.md index 8be890ef75f..bfead075135 100644 --- a/src/ripple/overlay/README.md +++ b/src/ripple/overlay/README.md @@ -343,10 +343,11 @@ messages for the local and remote endpoints, and combine them to generate a uniq "fingerprint". By design, this fingerprint should be the same for both SSL/TLS endpoints. -That fingerprint, which is never shared over the wire (since each endpoint will -calculate it independently), is then signed by each server using its public -**`secp256k1`** node identity and the signature is transferred over the SSL/TLS -encrypted link during the protocol handshake phase. +That fingerprint is calculated by each endpoint independently, so the +fingerprint is never transmitted over the network. Each server then utilizes its +private key to sign the fingerprint. This is the same keypair that determines +the server's public `secp256k1` node identity. The signature is transferred over +the secure SSL/TLS encrypted link during the protocol's initial handshake phase. Each side of the link will verify that the provided signature is from the claimed public key against the session's unique fingerprint. If this signature check fails @@ -364,6 +365,50 @@ transferred between A and B and will not be able to intelligently tamper with th message stream between Alice and Bob, although she may be still be able to inject delays or terminate the link. +## Peer Connection Sequence + +The _PeerImp_ object can be constructed as either an outbound or an inbound peer. +The outbound peer is constructed by the _ConnectAttempt_ - the client side of +the connection. The inbound peer is constructed by the _InboundHandoff_ - +the server side of the connection. This differentiation of the peers matters only +in terms of the object construction. Once constructed, both inbound and outbound +peer play the same role. + +### Outbound Peer + +An outbound connection is initiated once a second by +the _OverlayImpl::Timer::on_timer()_ method. This method calls +_OverlayImpl::autoConnect()_, which in turn calls _OverlayImpl::connect()_ for +every outbound endpoint generated by _PeerFinder::autoconnect()_. _connect()_ +method constructs _ConnectAttempt_ object. _ConnectAttempt_ attempts to connect +to the provided endpoint and on a successful connection executes the client side +of the handshake protocol described above. If the handshake is successful then +the outbound _PeerImp_ object is constructed and passed to the overlay manager +_OverlayImpl_, which adds the object to the list of peers and children. The latter +maintains a list of objects which might be executing an asynchronous operation +and therefore have to be stopped on shutdown. The outbound _PeerImp_ sends +_TMStartProtocol_ message on start to instruct the connected inbound peer that +the outbound peer is ready to receive the protocol messages. + +### Inbound Peer + +Construction of the inbound peer is more involved. A multi protocol-server, +_ServerImpl_ located in _src/ripple/server_ module, maintains multiple configured +listening ports. Each listening port allows for multiple protocols including HTTP, +HTTP/S, WebSocket, Secure WebSocket, and the Peer protocol. For simplicity this +sequence describes only the Peer protocol. _ServerImpl_ constructs +_Door_ object for each configured protocol. Each instance of the _Door_ object +accepts connections on the configured port. On a successful connection the _Door_ +constructs _SSLHTTPPeer_ object since the Peer protocol always uses SSL +connection. _SSLHTTPPeer_ executes the SSL handshake. If the handshake is successful +then a server handler, _ServerHandlerImpl_ located in _src/ripple/src/impl_, hands off +the connection to the _OverlayImpl::onHandoff()_ method. _onHandoff()_ method +validates the client's HTTP handshake request described above. If the request is +valid then the _InboundHandoff_ object is constructed. _InboundHandoff_ sends +HTTP response to the connected client, constructs the inbound _PeerImp_ object, +and passes it to the overlay manager _OverlayImpl_, which adds the object to +the list of peers and children. Once the inbound _PeerImp_ receives +_TMStartProtocol_ message, it starts sending the protocol messages. # Ripple Clustering # diff --git a/src/ripple/overlay/ReduceRelayCommon.h b/src/ripple/overlay/ReduceRelayCommon.h index 3b87c3c8c13..8289e467e65 100644 --- a/src/ripple/overlay/ReduceRelayCommon.h +++ b/src/ripple/overlay/ReduceRelayCommon.h @@ -24,6 +24,10 @@ namespace ripple { +// Blog post explaining the rationale behind reduction of flooding gossip +// protocol: +// https://xrpl.org/blog/2021/message-routing-optimizations-pt-1-proposal-validation-relaying.html + namespace reduce_relay { // Peer's squelch is limited in time to diff --git a/src/ripple/overlay/impl/InboundHandoff.cpp b/src/ripple/overlay/impl/InboundHandoff.cpp new file mode 100644 index 00000000000..1f45e1d37a7 --- /dev/null +++ b/src/ripple/overlay/impl/InboundHandoff.cpp @@ -0,0 +1,185 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +#include + +namespace ripple { + +InboundHandoff::InboundHandoff( + Application& app, + id_t id, + std::shared_ptr const& slot, + http_request_type&& request, + PublicKey const& publicKey, + ProtocolVersion protocol, + Resource::Consumer consumer, + std::unique_ptr&& stream_ptr, + OverlayImpl& overlay) + : OverlayImpl::Child(overlay) + , app_(app) + , id_(id) + , sink_( + app_.journal("Peer"), + [id]() { + std::stringstream ss; + ss << "[" << std::setfill('0') << std::setw(3) << id << "] "; + return ss.str(); + }()) + , journal_(sink_) + , stream_ptr_(std::move(stream_ptr)) + , strand_(stream_ptr_->next_layer().socket().get_executor()) + , remote_address_(slot->remote_endpoint()) + , protocol_(protocol) + , publicKey_(publicKey) + , usage_(consumer) + , slot_(slot) + , request_(std::move(request)) +{ +} + +void +InboundHandoff::run() +{ + if (!strand_.running_in_this_thread()) + return post( + strand_, std::bind(&InboundHandoff::run, shared_from_this())); + sendResponse(); +} + +void +InboundHandoff::stop() +{ + if (!strand_.running_in_this_thread()) + return post( + strand_, std::bind(&InboundHandoff::stop, shared_from_this())); + if (stream_ptr_->next_layer().socket().is_open()) + { + JLOG(journal_.debug()) << "Stop"; + } + close(); +} + +void +InboundHandoff::sendResponse() +{ + auto const sharedValue = makeSharedValue(*stream_ptr_, journal_); + // This shouldn't fail since we already computed + // the shared value successfully in OverlayImpl + if (!sharedValue) + return fail("makeSharedValue: Unexpected failure"); + + JLOG(journal_.info()) << "Protocol: " << to_string(protocol_); + JLOG(journal_.info()) << "Public Key: " + << toBase58(TokenType::NodePublic, publicKey_); + + auto write_buffer = std::make_shared(); + + boost::beast::ostream(*write_buffer) << makeResponse( + !overlay_.peerFinder().config().peerPrivate, + request_, + overlay_.setup().public_ip, + remote_address_.address(), + *sharedValue, + overlay_.setup().networkID, + protocol_, + app_); + + // Write the whole buffer and only start protocol when that's done. + boost::asio::async_write( + *stream_ptr_, + write_buffer->data(), + boost::asio::transfer_all(), + bind_executor( + strand_, + [this, write_buffer, self = shared_from_this()]( + error_code ec, std::size_t bytes_transferred) { + if (!stream_ptr_->next_layer().socket().is_open()) + return; + if (ec == boost::asio::error::operation_aborted) + return; + if (ec) + return fail("onWriteResponse", ec); + if (write_buffer->size() == bytes_transferred) + return createPeer(); + return fail("Failed to write header"); + })); +} + +void +InboundHandoff::fail(std::string const& name, error_code const& ec) +{ + if (socket().is_open()) + { + JLOG(journal_.warn()) + << name << " from " << toBase58(TokenType::NodePublic, publicKey_) + << " at " << remote_address_.to_string() << ": " << ec.message(); + } + close(); +} + +void +InboundHandoff::fail(std::string const& reason) +{ + if (journal_.active(beast::severities::kWarning) && socket().is_open()) + { + auto const n = app_.cluster().member(publicKey_); + JLOG(journal_.warn()) + << (n ? remote_address_.to_string() : *n) << " failed: " << reason; + } + close(); +} + +void +InboundHandoff::close() +{ + if (socket().is_open()) + { + socket().close(); + JLOG(journal_.debug()) << "Closed"; + } +} + +void +InboundHandoff::createPeer() +{ + auto peer = std::make_shared( + app_, + id_, + slot_, + std::move(request_), + publicKey_, + protocol_, + usage_, + std::move(stream_ptr_), + overlay_); + + overlay_.add_active(peer); +} + +InboundHandoff::socket_type& +InboundHandoff::socket() const +{ + return stream_ptr_->next_layer().socket(); +} + +} // namespace ripple \ No newline at end of file diff --git a/src/ripple/overlay/impl/InboundHandoff.h b/src/ripple/overlay/impl/InboundHandoff.h new file mode 100644 index 00000000000..3f3154c3a8f --- /dev/null +++ b/src/ripple/overlay/impl/InboundHandoff.h @@ -0,0 +1,102 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED +#define RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED + +#include + +namespace ripple { + +/** Sends HTTP response. Instantiates the inbound peer + * once the response is sent. Maintains all data members + * required for the inbound peer instantiation. + */ +class InboundHandoff : public OverlayImpl::Child, + public std::enable_shared_from_this +{ +private: + using error_code = boost::system::error_code; + using socket_type = boost::asio::ip::tcp::socket; + using middle_type = boost::beast::tcp_stream; + using stream_type = boost::beast::ssl_stream; + using id_t = Peer::id_t; + Application& app_; + id_t const id_; + beast::WrappedSink sink_; + beast::Journal const journal_; + std::unique_ptr stream_ptr_; + boost::asio::strand strand_; + beast::IP::Endpoint const remote_address_; + ProtocolVersion protocol_; + PublicKey const publicKey_; + Resource::Consumer usage_; + std::shared_ptr const slot_; + http_request_type request_; + +public: + virtual ~InboundHandoff() override = default; + + InboundHandoff( + Application& app, + id_t id, + std::shared_ptr const& slot, + http_request_type&& request, + PublicKey const& publicKey, + ProtocolVersion protocol, + Resource::Consumer consumer, + std::unique_ptr&& stream_ptr, + OverlayImpl& overlay); + + // This class isn't meant to be copied + InboundHandoff(InboundHandoff const&) = delete; + InboundHandoff& + operator=(InboundHandoff const&) = delete; + + /** Start the handshake */ + void + run(); + /** Stop the child */ + void + stop() override; + +private: + /** Send upgrade response to the client */ + void + sendResponse(); + /** Instantiate and run the overlay peer */ + void + createPeer(); + /** Log and close */ + void + fail(std::string const& name, error_code const& ec); + /** Log and close */ + void + fail(std::string const& reason); + /** Close connection */ + void + close(); + /** Get underlying socket */ + socket_type& + socket() const; +}; + +} // namespace ripple + +#endif // RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED diff --git a/src/ripple/overlay/impl/Message.cpp b/src/ripple/overlay/impl/Message.cpp index b4cb1f192aa..1b434225501 100644 --- a/src/ripple/overlay/impl/Message.cpp +++ b/src/ripple/overlay/impl/Message.cpp @@ -70,7 +70,7 @@ Message::compress() using namespace ripple::compression; auto const messageBytes = buffer_.size() - headerBytes; - auto type = getType(buffer_.data()); + auto type = getType(); bool const compressible = [&] { if (messageBytes <= 70) @@ -221,9 +221,10 @@ Message::getBuffer(Compressed tryCompressed) } int -Message::getType(std::uint8_t const* in) const +Message::getType() const { - int type = (static_cast(*(in + 4)) << 8) + *(in + 5); + int type = + (static_cast(*(buffer_.data() + 4)) << 8) + *(buffer_.data() + 5); return type; } diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index 6ed046f0403..c48ab378cb3 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -279,7 +280,7 @@ OverlayImpl::onHandoff( } } - auto const peer = std::make_shared( + auto const ih = std::make_shared( app_, id, slot, @@ -290,18 +291,10 @@ OverlayImpl::onHandoff( std::move(stream_ptr), *this); { - // As we are not on the strand, run() must be called - // while holding the lock, otherwise new I/O can be - // queued after a call to stop(). std::lock_guard lock(mutex_); - { - auto const result = m_peers.emplace(peer->slot(), peer); - assert(result.second); - (void)result.second; - } - list_.emplace(peer.get(), peer); + list_.emplace(ih.get(), ih); - peer->run(); + ih->run(); } handoff.moved = true; return handoff; diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 0d9b9fc549b..dc23f3325f8 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -66,6 +66,30 @@ std::chrono::milliseconds constexpr peerHighLatency{300}; std::chrono::seconds constexpr peerTimerInterval{60}; } // namespace +std::string +closeReasonToString(protocol::TMCloseReason reason) +{ + switch (reason) + { + case protocol::TMCloseReason::crCHARGE_RESOURCES: + return "Charge: Resources"; + case protocol::TMCloseReason::crMALFORMED_HANDSHAKE1: + return "Malformed handshake data (1)"; + case protocol::TMCloseReason::crMALFORMED_HANDSHAKE2: + return "Malformed handshake data (2)"; + case protocol::TMCloseReason::crMALFORMED_HANDSHAKE3: + return "Malformed handshake data (3)"; + case protocol::TMCloseReason::crLARGE_SENDQUEUE: + return "Large send queue"; + case protocol::TMCloseReason::crNOT_USEFUL: + return "Not useful"; + case protocol::TMCloseReason::crPING_TIMEOUT: + return "Ping timeout"; + default: + return "Unknown reason"; + } +} + PeerImp::PeerImp( Application& app, id_t id, @@ -132,6 +156,11 @@ PeerImp::PeerImp( << " tx reduce-relay enabled " << txReduceRelayEnabled_ << " on " << remote_address_ << " " << id_; + if (auto member = app_.cluster().member(publicKey_)) + { + name_ = *member; + JLOG(journal_.info()) << "Cluster name: " << *member; + } } PeerImp::~PeerImp() @@ -182,7 +211,7 @@ PeerImp::run() closed = parseLedgerHash(iter->value()); if (!closed) - fail("Malformed handshake data (1)"); + fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE1); } if (auto const iter = headers_.find("Previous-Ledger"); @@ -191,11 +220,11 @@ PeerImp::run() previous = parseLedgerHash(iter->value()); if (!previous) - fail("Malformed handshake data (2)"); + fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE2); } if (previous && !closed) - fail("Malformed handshake data (3)"); + fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE3); { std::lock_guard sl(recentLock_); @@ -205,10 +234,7 @@ PeerImp::run() previousLedgerHash_ = *previous; } - if (inbound_) - doAccept(); - else - doProtocolStart(); + doProtocolStart(); // Anything else that needs to be done with the connection should be // done in doProtocolStart @@ -350,7 +376,7 @@ PeerImp::charge(Resource::Charge const& fee) { // Sever the connection overlay_.incPeerDisconnectCharges(); - fail("charge: Resources"); + fail(protocol::TMCloseReason::crCHARGE_RESOURCES); } } @@ -508,6 +534,8 @@ PeerImp::supportsFeature(ProtocolFeature f) const return protocol_ >= make_protocol(2, 2); case ProtocolFeature::LedgerReplay: return ledgerReplayEnabled_; + case ProtocolFeature::StartProtocol: + return protocol_ >= make_protocol(2, 3); } return false; } @@ -600,22 +628,34 @@ PeerImp::close() } void -PeerImp::fail(std::string const& reason) +PeerImp::fail(protocol::TMCloseReason reason) { if (!strand_.running_in_this_thread()) return post( strand_, std::bind( - (void (Peer::*)(std::string const&)) & PeerImp::fail, + (void (Peer::*)(protocol::TMCloseReason)) & PeerImp::fail, shared_from_this(), reason)); if (journal_.active(beast::severities::kWarning) && socket_.is_open()) { std::string const n = name(); JLOG(journal_.warn()) << (n.empty() ? remote_address_.to_string() : n) - << " failed: " << reason; + << " failed: " << closeReasonToString(reason); } - close(); + + // erase all outstanding messages except for the one + // currently being executed + if (send_queue_.size() > 1) + { + decltype(send_queue_) q({send_queue_.front()}); + send_queue_.swap(q); + } + + closeOnWriteComplete_ = true; + protocol::TMGracefulClose tmGC; + tmGC.set_reason(reason); + send(std::make_shared(tmGC, protocol::mtGRACEFUL_CLOSE)); } void @@ -707,7 +747,7 @@ PeerImp::onTimer(error_code const& ec) if (large_sendq_++ >= Tuning::sendqIntervals) { - fail("Large send queue"); + fail(protocol::TMCloseReason::crLARGE_SENDQUEUE); return; } @@ -726,7 +766,7 @@ PeerImp::onTimer(error_code const& ec) (duration > app_.config().MAX_UNKNOWN_TIME))) { overlay_.peerFinder().on_failure(slot_); - fail("Not useful"); + fail(protocol::TMCloseReason::crLARGE_SENDQUEUE); return; } } @@ -734,7 +774,7 @@ PeerImp::onTimer(error_code const& ec) // Already waiting for PONG if (lastPingSeq_) { - fail("Ping Timeout"); + fail(protocol::TMCloseReason::crPING_TIMEOUT); return; } @@ -766,71 +806,6 @@ PeerImp::onShutdown(error_code ec) } //------------------------------------------------------------------------------ -void -PeerImp::doAccept() -{ - assert(read_buffer_.size() == 0); - - JLOG(journal_.debug()) << "doAccept: " << remote_address_; - - auto const sharedValue = makeSharedValue(*stream_ptr_, journal_); - - // This shouldn't fail since we already computed - // the shared value successfully in OverlayImpl - if (!sharedValue) - return fail("makeSharedValue: Unexpected failure"); - - JLOG(journal_.info()) << "Protocol: " << to_string(protocol_); - JLOG(journal_.info()) << "Public Key: " - << toBase58(TokenType::NodePublic, publicKey_); - - if (auto member = app_.cluster().member(publicKey_)) - { - { - std::unique_lock lock{nameMutex_}; - name_ = *member; - } - JLOG(journal_.info()) << "Cluster name: " << *member; - } - - overlay_.activate(shared_from_this()); - - // XXX Set timer: connection is in grace period to be useful. - // XXX Set timer: connection idle (idle may vary depending on connection - // type.) - - auto write_buffer = std::make_shared(); - - boost::beast::ostream(*write_buffer) << makeResponse( - !overlay_.peerFinder().config().peerPrivate, - request_, - overlay_.setup().public_ip, - remote_address_.address(), - *sharedValue, - overlay_.setup().networkID, - protocol_, - app_); - - // Write the whole buffer and only start protocol when that's done. - boost::asio::async_write( - stream_, - write_buffer->data(), - boost::asio::transfer_all(), - bind_executor( - strand_, - [this, write_buffer, self = shared_from_this()]( - error_code ec, std::size_t bytes_transferred) { - if (!socket_.is_open()) - return; - if (ec == boost::asio::error::operation_aborted) - return; - if (ec) - return fail("onWriteResponse", ec); - if (write_buffer->size() == bytes_transferred) - return doProtocolStart(); - return fail("Failed to write header"); - })); -} std::string PeerImp::name() const @@ -854,39 +829,49 @@ PeerImp::doProtocolStart() { onReadMessage(error_code(), 0); - // Send all the validator lists that have been loaded - if (inbound_ && supportsFeature(ProtocolFeature::ValidatorListPropagation)) + bool supportedProtocol = supportsFeature(ProtocolFeature::StartProtocol); + + if (!inbound_) { - app_.validators().for_each_available( - [&](std::string const& manifest, - std::uint32_t version, - std::map const& blobInfos, - PublicKey const& pubKey, - std::size_t maxSequence, - uint256 const& hash) { - ValidatorList::sendValidatorList( - *this, - 0, - pubKey, - maxSequence, - version, - manifest, - blobInfos, - app_.getHashRouter(), - p_journal_); + // Instruct connected inbound peer to start sending + // protocol messages + if (supportedProtocol) + { + JLOG(journal_.debug()) + << "doProtocolStart(): outbound sending mtSTART_PROTOCOL to " + << remote_address_; + protocol::TMStartProtocol tmPS; + tmPS.set_starttime(std::chrono::duration_cast( + clock_type::now().time_since_epoch()) + .count()); + send(std::make_shared(tmPS, protocol::mtSTART_PROTOCOL)); + } + else + { + JLOG(journal_.debug()) << "doProtocolStart(): outbound connected " + "to an older protocol on " + << remote_address_ << " " << protocol_.first + << " " << protocol_.second; + } - // Don't send it next time. - app_.getHashRouter().addSuppressionPeer(hash, id_); - }); - } + if (auto m = overlay_.getManifestsMessage()) + send(m); - if (auto m = overlay_.getManifestsMessage()) - send(m); - - // Request shard info from peer - protocol::TMGetPeerShardInfoV2 tmGPS; - tmGPS.set_relays(0); - send(std::make_shared(tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); + // Request shard info from peer + protocol::TMGetPeerShardInfoV2 tmGPS; + tmGPS.set_relays(0); + send(std::make_shared( + tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); + } + // Backward compatibility with the older protocols + else if (!supportedProtocol) + { + JLOG(journal_.debug()) + << "doProtocolStart(): inbound handling of an older protocol on " + << remote_address_ << " " << protocol_.first << " " + << protocol_.second; + onStartProtocol(); + } setTimer(); } @@ -954,7 +939,11 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) if (!socket_.is_open()) return; if (ec == boost::asio::error::operation_aborted) + { + if (closeOnWriteComplete_) + close(); return; + } if (ec) return fail("onWriteMessage", ec); if (auto stream = journal_.trace()) @@ -968,6 +957,11 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) metrics_.sent.add_message(bytes_transferred); assert(!send_queue_.empty()); + if (send_queue_.front()->getType() == protocol::mtGRACEFUL_CLOSE) + { + close(); + return; + } send_queue_.pop(); if (!send_queue_.empty()) { @@ -2947,6 +2941,69 @@ PeerImp::onMessage(std::shared_ptr const& m) << "onMessage: TMSquelch " << slice << " " << id() << " " << duration; } +void +PeerImp::onStartProtocol() +{ + JLOG(journal_.debug()) << "onStartProtocol(): " << remote_address_; + // Send all the validator lists that have been loaded + if (supportsFeature(ProtocolFeature::ValidatorListPropagation)) + { + app_.validators().for_each_available( + [&](std::string const& manifest, + std::uint32_t version, + std::map const& blobInfos, + PublicKey const& pubKey, + std::size_t maxSequence, + uint256 const& hash) { + ValidatorList::sendValidatorList( + *this, + 0, + pubKey, + maxSequence, + version, + manifest, + blobInfos, + app_.getHashRouter(), + p_journal_); + + // Don't send it next time. + app_.getHashRouter().addSuppressionPeer(hash, id_); + }); + } + + if (auto m = overlay_.getManifestsMessage()) + send(m); + + // Request shard info from peer + protocol::TMGetPeerShardInfoV2 tmGPS; + tmGPS.set_relays(0); + send(std::make_shared(tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); +} + +void +PeerImp::onMessage(std::shared_ptr const& m) +{ + JLOG(journal_.debug()) << "onMessage(TMStartProtocol): " << remote_address_; + onStartProtocol(); +} + +void +PeerImp::onMessage(const std::shared_ptr& m) +{ + using on_message_fn = + void (PeerImp::*)(std::shared_ptr const&); + if (!strand_.running_in_this_thread()) + return post( + strand_, + std::bind( + (on_message_fn)&PeerImp::onMessage, shared_from_this(), m)); + + JLOG(journal_.info()) << "got graceful close from: " << remote_address_ + << " reason: " << closeReasonToString(m->reason()); + + close(); +} + //-------------------------------------------------------------------------- void diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index 710ab4d74d6..d922e757946 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -180,6 +180,8 @@ class PeerImp : public Peer, bool vpReduceRelayEnabled_ = false; bool ledgerReplayEnabled_ = false; LedgerReplayMsgHandler ledgerReplayMsgHandler_; + // close connection when async write is complete + bool closeOnWriteComplete_ = false; friend class OverlayImpl; @@ -235,7 +237,7 @@ class PeerImp : public Peer, /** Create outgoing, handshaked peer. */ // VFALCO legacyPublicKey should be implied by the Slot - template + template PeerImp( Application& app, std::unique_ptr&& stream_ptr, @@ -413,7 +415,7 @@ class PeerImp : public Peer, isHighLatency() const override; void - fail(std::string const& reason); + fail(protocol::TMCloseReason reason); // Return any known shard info from this peer and its sub peers [[nodiscard]] hash_map const @@ -458,9 +460,6 @@ class PeerImp : public Peer, void onShutdown(error_code ec); - void - doAccept(); - std::string name() const; @@ -584,6 +583,10 @@ class PeerImp : public Peer, onMessage(std::shared_ptr const& m); void onMessage(std::shared_ptr const& m); + void + onMessage(std::shared_ptr const& m); + void + onMessage(std::shared_ptr const& m); private: //-------------------------------------------------------------------------- @@ -642,6 +645,9 @@ class PeerImp : public Peer, void processLedgerRequest(std::shared_ptr const& m); + + void + onStartProtocol(); }; //------------------------------------------------------------------------------ diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index d6fb14bc78c..6071a621db5 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -112,6 +112,10 @@ protocolMessageName(int type) return "get_peer_shard_info_v2"; case protocol::mtPEER_SHARD_INFO_V2: return "peer_shard_info_v2"; + case protocol::mtSTART_PROTOCOL: + return "start_protocol"; + case protocol::mtGRACEFUL_CLOSE: + return "graceful_close"; default: break; } @@ -492,6 +496,14 @@ invokeProtocolMessage( success = detail::invoke( *header, buffers, handler); break; + case protocol::mtSTART_PROTOCOL: + success = detail::invoke( + *header, buffers, handler); + break; + case protocol::mtGRACEFUL_CLOSE: + success = detail::invoke( + *header, buffers, handler); + break; default: handler.onMessageUnknown(header->message_type); success = true; diff --git a/src/ripple/overlay/impl/ProtocolVersion.cpp b/src/ripple/overlay/impl/ProtocolVersion.cpp index fbd48474420..8325f6d32fb 100644 --- a/src/ripple/overlay/impl/ProtocolVersion.cpp +++ b/src/ripple/overlay/impl/ProtocolVersion.cpp @@ -37,7 +37,8 @@ namespace ripple { constexpr ProtocolVersion const supportedProtocolList[] { {2, 1}, - {2, 2} + {2, 2}, + {2, 3} }; // clang-format on diff --git a/src/ripple/proto/ripple.proto b/src/ripple/proto/ripple.proto index d116b992a90..5ea0fcba450 100644 --- a/src/ripple/proto/ripple.proto +++ b/src/ripple/proto/ripple.proto @@ -33,6 +33,8 @@ enum MessageType mtPEER_SHARD_INFO_V2 = 62; mtHAVE_TRANSACTIONS = 63; mtTRANSACTIONS = 64; + mtSTART_PROTOCOL = 65; + mtGRACEFUL_CLOSE = 66; } // token, iterations, target, challenge = issue demand for proof of work @@ -452,3 +454,24 @@ message TMHaveTransactions repeated bytes hashes = 1; } +message TMStartProtocol +{ + required uint64 startTime = 1; +} + +enum TMCloseReason +{ + crMALFORMED_HANDSHAKE1 = 1; + crMALFORMED_HANDSHAKE2 = 2; + crMALFORMED_HANDSHAKE3 = 3; + crCHARGE_RESOURCES = 4; + crLARGE_SENDQUEUE = 5; + crNOT_USEFUL = 6; + crPING_TIMEOUT = 7; +} + +message TMGracefulClose +{ + required TMCloseReason reason = 1; +} + diff --git a/src/ripple/protocol/ErrorCodes.h b/src/ripple/protocol/ErrorCodes.h index 87323b0dea8..8319b69c8c2 100644 --- a/src/ripple/protocol/ErrorCodes.h +++ b/src/ripple/protocol/ErrorCodes.h @@ -78,7 +78,7 @@ enum error_code_i { // unused 27, // unused 28, rpcTXN_NOT_FOUND = 29, - // unused 30, + rpcINVALID_HOTWALLET = 30, // Malformed command rpcINVALID_PARAMS = 31, diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index 933cca3a7e9..a0f43df8b64 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -33,7 +33,7 @@ namespace BuildInfo { // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ // clang-format off -char const* const versionString = "2.0.0-b1" +char const* const versionString = "2.0.0-b2" // clang-format on #if defined(DEBUG) || defined(SANITIZER) diff --git a/src/ripple/protocol/impl/ErrorCodes.cpp b/src/ripple/protocol/impl/ErrorCodes.cpp index 603888827b2..319bd8e28c2 100644 --- a/src/ripple/protocol/impl/ErrorCodes.cpp +++ b/src/ripple/protocol/impl/ErrorCodes.cpp @@ -76,6 +76,7 @@ constexpr static ErrorInfo unorderedErrorInfos[]{ {rpcINTERNAL, "internal", "Internal error.", 500}, {rpcINVALID_LGR_RANGE, "invalidLgrRange", "Ledger range is invalid.", 400}, {rpcINVALID_PARAMS, "invalidParams", "Invalid parameters.", 400}, + {rpcINVALID_HOTWALLET, "invalidHotWallet", "Invalid hotwallet.", 400}, {rpcISSUE_MALFORMED, "issueMalformed", "Issue is malformed.", 400}, {rpcJSON_RPC, "json_rpc", "JSON-RPC transport error.", 500}, {rpcLGR_IDXS_INVALID, "lgrIdxsInvalid", "Ledger indexes invalid.", 400}, diff --git a/src/ripple/protocol/impl/UintTypes.cpp b/src/ripple/protocol/impl/UintTypes.cpp index ff2644b085b..821e81238b0 100644 --- a/src/ripple/protocol/impl/UintTypes.cpp +++ b/src/ripple/protocol/impl/UintTypes.cpp @@ -93,14 +93,8 @@ to_currency(Currency& currency, std::string const& code) currency = beast::zero; - std::transform( - code.begin(), - code.end(), - currency.begin() + detail::isoCodeOffset, - [](auto c) { - return static_cast( - ::toupper(static_cast(c))); - }); + std::copy( + code.begin(), code.end(), currency.begin() + detail::isoCodeOffset); return true; } diff --git a/src/ripple/rpc/handlers/GatewayBalances.cpp b/src/ripple/rpc/handlers/GatewayBalances.cpp index 77cec496ed0..fc6a7b49fd8 100644 --- a/src/ripple/rpc/handlers/GatewayBalances.cpp +++ b/src/ripple/rpc/handlers/GatewayBalances.cpp @@ -78,6 +78,12 @@ doGatewayBalances(RPC::JsonContext& context) result[jss::account] = toBase58(accountID); + if (context.apiVersion > 1u && !ledger->exists(keylet::account(accountID))) + { + RPC::inject_error(rpcACT_NOT_FOUND, result); + return result; + } + // Parse the specified hotwallet(s), if any std::set hotWallets; @@ -116,7 +122,18 @@ doGatewayBalances(RPC::JsonContext& context) if (!valid) { - result[jss::error] = "invalidHotWallet"; + // The documentation states that invalidParams is used when + // One or more fields are specified incorrectly. + // invalidHotwallet should be used when the account exists, but does + // not have currency issued by the account from the request. + if (context.apiVersion < 2u) + { + RPC::inject_error(rpcINVALID_HOTWALLET, result); + } + else + { + RPC::inject_error(rpcINVALID_PARAMS, result); + } return result; } } diff --git a/src/ripple/rpc/handlers/PayChanClaim.cpp b/src/ripple/rpc/handlers/PayChanClaim.cpp index 6353124cb39..23a4041bb35 100644 --- a/src/ripple/rpc/handlers/PayChanClaim.cpp +++ b/src/ripple/rpc/handlers/PayChanClaim.cpp @@ -55,7 +55,8 @@ doChannelAuthorize(RPC::JsonContext& context) return RPC::missing_field_error(jss::secret); Json::Value result; - auto const [pk, sk] = RPC::keypairForSignature(params, result); + auto const [pk, sk] = + RPC::keypairForSignature(params, result, context.apiVersion); if (RPC::contains_error(result)) return result; diff --git a/src/ripple/rpc/impl/RPCHelpers.cpp b/src/ripple/rpc/impl/RPCHelpers.cpp index 898755bd5ab..f082f8913ca 100644 --- a/src/ripple/rpc/impl/RPCHelpers.cpp +++ b/src/ripple/rpc/impl/RPCHelpers.cpp @@ -847,7 +847,10 @@ getSeedFromRPC(Json::Value const& params, Json::Value& error) } std::pair -keypairForSignature(Json::Value const& params, Json::Value& error) +keypairForSignature( + Json::Value const& params, + Json::Value& error, + unsigned int apiVersion) { bool const has_key_type = params.isMember(jss::key_type); @@ -901,7 +904,10 @@ keypairForSignature(Json::Value const& params, Json::Value& error) if (!keyType) { - error = RPC::invalid_field_error(jss::key_type); + if (apiVersion > 1u) + error = RPC::make_error(rpcBAD_KEY_TYPE); + else + error = RPC::invalid_field_error(jss::key_type); return {}; } diff --git a/src/ripple/rpc/impl/RPCHelpers.h b/src/ripple/rpc/impl/RPCHelpers.h index 5fa7ae804a2..eb02e6ea37a 100644 --- a/src/ripple/rpc/impl/RPCHelpers.h +++ b/src/ripple/rpc/impl/RPCHelpers.h @@ -208,9 +208,6 @@ getSeedFromRPC(Json::Value const& params, Json::Value& error); std::optional parseRippleLibSeed(Json::Value const& params); -std::pair -keypairForSignature(Json::Value const& params, Json::Value& error); - /** * API version numbers used in API version 1 */ @@ -295,6 +292,11 @@ getAPIVersionNumber(const Json::Value& value, bool betaEnabled); std::variant, Json::Value> getLedgerByContext(RPC::JsonContext& context); +std::pair +keypairForSignature( + Json::Value const& params, + Json::Value& error, + unsigned int apiVersion = apiVersionIfUnspecified); /** Helper to parse submit_mode parameter to RPC submit. * * @param params RPC parameters diff --git a/src/test/app/PayChan_test.cpp b/src/test/app/PayChan_test.cpp index f8162dd107d..02a13de5c21 100644 --- a/src/test/app/PayChan_test.cpp +++ b/src/test/app/PayChan_test.cpp @@ -24,9 +24,9 @@ #include #include #include -#include - +#include #include +#include namespace ripple { namespace test { @@ -1063,6 +1063,47 @@ struct PayChan_test : public beast::unit_test::suite bob.human()); } + void + testAccountChannelAuthorize(FeatureBitset features) + { + using namespace jtx; + using namespace std::literals::chrono_literals; + + Env env{*this, features}; + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const charlie = Account("charlie", KeyType::ed25519); + env.fund(XRP(10000), alice, bob, charlie); + auto const pk = alice.pk(); + auto const settleDelay = 3600s; + auto const channelFunds = XRP(1000); + auto const chan1Str = to_string(channel(alice, bob, env.seq(alice))); + env(create(alice, bob, channelFunds, settleDelay, pk)); + env.close(); + + Json::Value args{Json::objectValue}; + args[jss::channel_id] = chan1Str; + args[jss::key_type] = "ed255191"; + args[jss::seed] = "snHq1rzQoN2qiUkC3XF5RyxBzUtN"; + args[jss::amount] = 51110000; + + // test for all api versions + for (auto apiVersion = RPC::apiMinimumSupportedVersion; + apiVersion <= RPC::apiBetaVersion; + ++apiVersion) + { + testcase( + "PayChan Channel_Auth RPC Api " + std::to_string(apiVersion)); + args[jss::api_version] = apiVersion; + auto const rs = env.rpc( + "json", + "channel_authorize", + args.toStyledString())[jss::result]; + auto const error = apiVersion < 2u ? "invalidParams" : "badKeyType"; + BEAST_EXPECT(rs[jss::error] == error); + } + } + void testAuthVerifyRPC(FeatureBitset features) { @@ -2042,6 +2083,7 @@ struct PayChan_test : public beast::unit_test::suite testAccountChannelsRPC(features); testAccountChannelsRPCMarkers(features); testAccountChannelsRPCSenderOnly(features); + testAccountChannelAuthorize(features); testAuthVerifyRPC(features); testOptionalFields(features); testMalformedPK(features); diff --git a/src/test/jtx/ManualTimeKeeper.h b/src/test/jtx/ManualTimeKeeper.h index 838f2c1398f..f3adb29b5f0 100644 --- a/src/test/jtx/ManualTimeKeeper.h +++ b/src/test/jtx/ManualTimeKeeper.h @@ -21,45 +21,30 @@ #define RIPPLE_TEST_MANUALTIMEKEEPER_H_INCLUDED #include -#include +#include namespace ripple { namespace test { class ManualTimeKeeper : public TimeKeeper { -public: - ManualTimeKeeper(); - - void - run(std::vector const& servers) override; +private: + std::atomic now_{}; - time_point - now() const override; +public: + ManualTimeKeeper() = default; time_point - closeTime() const override; + now() const override + { + return now_.load(); + } void - adjustCloseTime(std::chrono::duration amount) override; - - std::chrono::duration - nowOffset() const override; - - std::chrono::duration - closeOffset() const override; - - void - set(time_point now); - -private: - // Adjust system_clock::time_point for NetClock epoch - static time_point - adjust(std::chrono::system_clock::time_point when); - - std::mutex mutable mutex_; - std::chrono::duration closeOffset_; - time_point now_; + set(time_point now) + { + now_.store(now); + } }; } // namespace test diff --git a/src/test/jtx/impl/ManualTimeKeeper.cpp b/src/test/jtx/impl/ManualTimeKeeper.cpp deleted file mode 100644 index 72ceaa30bc0..00000000000 --- a/src/test/jtx/impl/ManualTimeKeeper.cpp +++ /dev/null @@ -1,95 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2015 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { -namespace test { - -using namespace std::chrono_literals; - -ManualTimeKeeper::ManualTimeKeeper() : closeOffset_{}, now_(0s) -{ -} - -void -ManualTimeKeeper::run(std::vector const& servers) -{ -} - -auto -ManualTimeKeeper::now() const -> time_point -{ - std::lock_guard lock(mutex_); - return now_; -} - -auto -ManualTimeKeeper::closeTime() const -> time_point -{ - std::lock_guard lock(mutex_); - return now_ + closeOffset_; -} - -void -ManualTimeKeeper::adjustCloseTime(std::chrono::duration amount) -{ - // Copied from TimeKeeper::adjustCloseTime - using namespace std::chrono; - auto const s = amount.count(); - std::lock_guard lock(mutex_); - // Take large offsets, ignore small offsets, - // push the close time towards our wall time. - if (s > 1) - closeOffset_ += seconds((s + 3) / 4); - else if (s < -1) - closeOffset_ += seconds((s - 3) / 4); - else - closeOffset_ = (closeOffset_ * 3) / 4; -} - -std::chrono::duration -ManualTimeKeeper::nowOffset() const -{ - return {}; -} - -std::chrono::duration -ManualTimeKeeper::closeOffset() const -{ - std::lock_guard lock(mutex_); - return closeOffset_; -} - -void -ManualTimeKeeper::set(time_point now) -{ - std::lock_guard lock(mutex_); - now_ = now; -} - -auto -ManualTimeKeeper::adjust(std::chrono::system_clock::time_point when) - -> time_point -{ - return time_point(std::chrono::duration_cast( - when.time_since_epoch() - days(10957))); -} -} // namespace test -} // namespace ripple diff --git a/src/test/overlay/ProtocolVersion_test.cpp b/src/test/overlay/ProtocolVersion_test.cpp index a5a26fe74ec..3bfba5099f4 100644 --- a/src/test/overlay/ProtocolVersion_test.cpp +++ b/src/test/overlay/ProtocolVersion_test.cpp @@ -88,7 +88,7 @@ class ProtocolVersion_test : public beast::unit_test::suite BEAST_EXPECT( negotiateProtocolVersion( "RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") == - make_protocol(2, 2)); + make_protocol(2, 3)); BEAST_EXPECT( negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt); diff --git a/src/test/overlay/compression_test.cpp b/src/test/overlay/compression_test.cpp index 3b61b2b3a09..81f21258e30 100644 --- a/src/test/overlay/compression_test.cpp +++ b/src/test/overlay/compression_test.cpp @@ -20,9 +20,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -227,8 +227,7 @@ class compression_test : public beast::unit_test::suite auto transaction = std::make_shared(); transaction->set_rawtransaction(usdTxBlob); transaction->set_status(protocol::tsNEW); - auto tk = make_TimeKeeper(logs.journal("TimeKeeper")); - transaction->set_receivetimestamp(tk->now().time_since_epoch().count()); + transaction->set_receivetimestamp(rand_int()); transaction->set_deferred(true); return transaction; @@ -263,19 +262,23 @@ class compression_test : public beast::unit_test::suite ledgerData->set_error(protocol::TMReplyError::reNO_LEDGER); ledgerData->mutable_nodes()->Reserve(n); uint256 parentHash(0); + + NetClock::duration const resolution{10}; + NetClock::time_point ct{resolution}; + for (int i = 0; i < n; i++) { LedgerInfo info; - auto tk = make_TimeKeeper(logs.journal("TimeKeeper")); info.seq = i; - info.parentCloseTime = tk->now(); + info.parentCloseTime = ct; info.hash = ripple::sha512Half(i); info.txHash = ripple::sha512Half(i + 1); info.accountHash = ripple::sha512Half(i + 2); info.parentHash = parentHash; info.drops = XRPAmount(10); - info.closeTimeResolution = tk->now().time_since_epoch(); - info.closeTime = tk->now(); + info.closeTimeResolution = resolution; + info.closeTime = ct; + ct += resolution; parentHash = ledgerHash(info); Serializer nData; ripple::addRaw(info, nData); @@ -341,7 +344,7 @@ class compression_test : public beast::unit_test::suite Serializer s1; st.add(s1); list->set_signature(s1.data(), s1.size()); - list->set_blob(strHex(s.getString())); + list->set_blob(strHex(s.slice())); return list; } @@ -375,7 +378,7 @@ class compression_test : public beast::unit_test::suite st.add(s1); auto& blob = *list->add_blobs(); blob.set_signature(s1.data(), s1.size()); - blob.set_blob(strHex(s.getString())); + blob.set_blob(strHex(s.slice())); return list; } diff --git a/src/test/rpc/GatewayBalances_test.cpp b/src/test/rpc/GatewayBalances_test.cpp index c14ec0f043c..091b9f51686 100644 --- a/src/test/rpc/GatewayBalances_test.cpp +++ b/src/test/rpc/GatewayBalances_test.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -34,117 +35,155 @@ class GatewayBalances_test : public beast::unit_test::suite using namespace jtx; Env env(*this, features); + { + // Gateway account and assets + Account const alice{"alice"}; + env.fund(XRP(10000), "alice"); + auto USD = alice["USD"]; + auto CNY = alice["CNY"]; + auto JPY = alice["JPY"]; + + // Create a hotwallet + Account const hw{"hw"}; + env.fund(XRP(10000), "hw"); + env(trust(hw, USD(10000))); + env(trust(hw, JPY(10000))); + env(pay(alice, hw, USD(5000))); + env(pay(alice, hw, JPY(5000))); + + // Create some clients + Account const bob{"bob"}; + env.fund(XRP(10000), "bob"); + env(trust(bob, USD(100))); + env(trust(bob, CNY(100))); + env(pay(alice, bob, USD(50))); + + Account const charley{"charley"}; + env.fund(XRP(10000), "charley"); + env(trust(charley, CNY(500))); + env(trust(charley, JPY(500))); + env(pay(alice, charley, CNY(250))); + env(pay(alice, charley, JPY(250))); + + Account const dave{"dave"}; + env.fund(XRP(10000), "dave"); + env(trust(dave, CNY(100))); + env(pay(alice, dave, CNY(30))); + + // give the gateway an asset + env(trust(alice, charley["USD"](50))); + env(pay(charley, alice, USD(10))); + + // freeze dave + env(trust(alice, dave["CNY"](0), dave, tfSetFreeze)); + + env.close(); + + auto wsc = makeWSClient(env.app().config()); + + Json::Value qry; + qry[jss::account] = alice.human(); + qry[jss::hotwallet] = hw.human(); + + auto jv = wsc->invoke("gateway_balances", qry); + expect(jv[jss::status] == "success"); + if (wsc->version() == 2) + { + expect(jv.isMember(jss::jsonrpc) && jv[jss::jsonrpc] == "2.0"); + expect( + jv.isMember(jss::ripplerpc) && jv[jss::ripplerpc] == "2.0"); + expect(jv.isMember(jss::id) && jv[jss::id] == 5); + } + + auto const& result = jv[jss::result]; + expect(result[jss::account] == alice.human()); + expect(result[jss::status] == "success"); + + { + auto const& balances = result[jss::balances]; + expect(balances.isObject(), "balances is not an object"); + expect(balances.size() == 1, "balances size is not 1"); + + auto const& hwBalance = balances[hw.human()]; + expect(hwBalance.isArray(), "hwBalance is not an array"); + expect(hwBalance.size() == 2); + auto c1 = hwBalance[0u][jss::currency]; + auto c2 = hwBalance[1u][jss::currency]; + expect(c1 == "USD" || c2 == "USD"); + expect(c1 == "JPY" || c2 == "JPY"); + expect( + hwBalance[0u][jss::value] == "5000" && + hwBalance[1u][jss::value] == "5000"); + } + + { + auto const& fBalances = result[jss::frozen_balances]; + expect(fBalances.isObject()); + expect(fBalances.size() == 1); + + auto const& fBal = fBalances[dave.human()]; + expect(fBal.isArray()); + expect(fBal.size() == 1); + expect(fBal[0u].isObject()); + expect(fBal[0u][jss::currency] == "CNY"); + expect(fBal[0u][jss::value] == "30"); + } + + { + auto const& assets = result[jss::assets]; + expect(assets.isObject(), "assets it not an object"); + expect(assets.size() == 1, "assets size is not 1"); + + auto const& cAssets = assets[charley.human()]; + expect(cAssets.isArray()); + expect(cAssets.size() == 1); + expect(cAssets[0u][jss::currency] == "USD"); + expect(cAssets[0u][jss::value] == "10"); + } + + { + auto const& obligations = result[jss::obligations]; + expect(obligations.isObject(), "obligations is not an object"); + expect(obligations.size() == 3); + expect(obligations["CNY"] == "250"); + expect(obligations["JPY"] == "250"); + expect(obligations["USD"] == "50"); + } + } + } + + void + testGWBApiVersions(FeatureBitset features) + { + using namespace std::chrono_literals; + using namespace jtx; + Env env(*this, features); + // Gateway account and assets Account const alice{"alice"}; - env.fund(XRP(10000), "alice"); - auto USD = alice["USD"]; - auto CNY = alice["CNY"]; - auto JPY = alice["JPY"]; - - // Create a hotwallet + env.fund(XRP(10000), alice); Account const hw{"hw"}; - env.fund(XRP(10000), "hw"); - env(trust(hw, USD(10000))); - env(trust(hw, JPY(10000))); - env(pay(alice, hw, USD(5000))); - env(pay(alice, hw, JPY(5000))); - - // Create some clients - Account const bob{"bob"}; - env.fund(XRP(10000), "bob"); - env(trust(bob, USD(100))); - env(trust(bob, CNY(100))); - env(pay(alice, bob, USD(50))); - - Account const charley{"charley"}; - env.fund(XRP(10000), "charley"); - env(trust(charley, CNY(500))); - env(trust(charley, JPY(500))); - env(pay(alice, charley, CNY(250))); - env(pay(alice, charley, JPY(250))); - - Account const dave{"dave"}; - env.fund(XRP(10000), "dave"); - env(trust(dave, CNY(100))); - env(pay(alice, dave, CNY(30))); - - // give the gateway an asset - env(trust(alice, charley["USD"](50))); - env(pay(charley, alice, USD(10))); - - // freeze dave - env(trust(alice, dave["CNY"](0), dave, tfSetFreeze)); - + env.fund(XRP(10000), hw); env.close(); auto wsc = makeWSClient(env.app().config()); - Json::Value qry; - qry[jss::account] = alice.human(); - qry[jss::hotwallet] = hw.human(); - - auto jv = wsc->invoke("gateway_balances", qry); - expect(jv[jss::status] == "success"); - if (wsc->version() == 2) - { - expect(jv.isMember(jss::jsonrpc) && jv[jss::jsonrpc] == "2.0"); - expect(jv.isMember(jss::ripplerpc) && jv[jss::ripplerpc] == "2.0"); - expect(jv.isMember(jss::id) && jv[jss::id] == 5); - } - - auto const& result = jv[jss::result]; - expect(result[jss::account] == alice.human()); - expect(result[jss::status] == "success"); - - { - auto const& balances = result[jss::balances]; - expect(balances.isObject(), "balances is not an object"); - expect(balances.size() == 1, "balances size is not 1"); - - auto const& hwBalance = balances[hw.human()]; - expect(hwBalance.isArray(), "hwBalance is not an array"); - expect(hwBalance.size() == 2); - auto c1 = hwBalance[0u][jss::currency]; - auto c2 = hwBalance[1u][jss::currency]; - expect(c1 == "USD" || c2 == "USD"); - expect(c1 == "JPY" || c2 == "JPY"); - expect( - hwBalance[0u][jss::value] == "5000" && - hwBalance[1u][jss::value] == "5000"); - } - - { - auto const& fBalances = result[jss::frozen_balances]; - expect(fBalances.isObject()); - expect(fBalances.size() == 1); - - auto const& fBal = fBalances[dave.human()]; - expect(fBal.isArray()); - expect(fBal.size() == 1); - expect(fBal[0u].isObject()); - expect(fBal[0u][jss::currency] == "CNY"); - expect(fBal[0u][jss::value] == "30"); - } + Json::Value qry2; + qry2[jss::account] = alice.human(); + qry2[jss::hotwallet] = "asdf"; + for (auto apiVersion = RPC::apiMinimumSupportedVersion; + apiVersion <= RPC::apiBetaVersion; + ++apiVersion) { - auto const& assets = result[jss::assets]; - expect(assets.isObject(), "assets it not an object"); - expect(assets.size() == 1, "assets size is not 1"); - - auto const& cAssets = assets[charley.human()]; - expect(cAssets.isArray()); - expect(cAssets.size() == 1); - expect(cAssets[0u][jss::currency] == "USD"); - expect(cAssets[0u][jss::value] == "10"); - } - - { - auto const& obligations = result[jss::obligations]; - expect(obligations.isObject(), "obligations is not an object"); - expect(obligations.size() == 3); - expect(obligations["CNY"] == "250"); - expect(obligations["JPY"] == "250"); - expect(obligations["USD"] == "50"); + qry2[jss::api_version] = apiVersion; + auto jv = wsc->invoke("gateway_balances", qry2); + expect(jv[jss::status] == "error"); + + auto response = jv[jss::result]; + auto const error = + apiVersion < 2u ? "invalidHotWallet" : "invalidParams"; + BEAST_EXPECT(response[jss::error] == error); } } @@ -207,8 +246,11 @@ class GatewayBalances_test : public beast::unit_test::suite { using namespace jtx; auto const sa = supported_amendments(); - testGWB(sa - featureFlowCross); - testGWB(sa); + for (auto feature : {sa - featureFlowCross, sa}) + { + testGWB(feature); + testGWBApiVersions(feature); + } testGWBOverflow(); }