Merge remote-tracking branch 'upstream/develop' into action

* upstream/develop:
  Set version to 1.12.0-rc1
  Set version to 1.12-b3
  Several changes to improve Consensus stability: (4505)
  Apply transaction batches in periodic intervals (4504)
  Asynchronously write batches to NuDB (4503)
  refactor: fix typo in FeeUnits.h (4644)
  Update Ubuntu build image (4650)
  test: add forAllApiVersions helper function (4611)
  add view updates for account SLEs (4629)
  Fix the package recipe for consumers of libxrpl (4631)
  refactor: improve checking of path lengths (4519)
  refactor: use C++20 function std::popcount (4389)
  fix(AMM): prevent orphaned objects, inconsistent ledger state: (4626)
  feat: support Concise Transaction Identifier (CTID) (XLS-37) (4418)

API-CHANGELOG.md

@@ -47,6 +47,28 @@ Additions are intended to be non-breaking (because they are purely additive).
  - Adds the [Clawback transaction type](https:/XRPLF/XRPL-Standards/blob/master/XLS-39d-clawback/README.md#331-clawback-transaction), containing these fields:
  - `Account`: The issuer of the asset being clawed back. Must also be the sender of the transaction.
  - `Amount`: The amount being clawed back, with the `Amount.issuer` being the token holder's address.
+- Adds [AMM](https:/XRPLF/XRPL-Standards/discussions/78) ([#4294](https:/XRPLF/rippled/pull/4294), [#4626](https:/XRPLF/rippled/pull/4626)) feature:
+ - Adds `amm_info` API to retrieve AMM information for a given tokens pair.
+ - Adds `AMMCreate` transaction type to create `AMM` instance.
+ - Adds `AMMDeposit` transaction type to deposit funds into `AMM` instance.
+ - Adds `AMMWithdraw` transaction type to withdraw funds from `AMM` instance.
+ - Adds `AMMVote` transaction type to vote for the trading fee of `AMM` instance.
+ - Adds `AMMBid` transaction type to bid for the Auction Slot of `AMM` instance.
+ - Adds `AMMDelete` transaction type to delete `AMM` instance.
+ - Adds `sfAMMID` to `AccountRoot` to indicate that the account is `AMM`'s account. `AMMID` is used to fetch `ltAMM`.
+ - Adds `lsfAMMNode` `TrustLine` flag to indicate that one side of the `TrustLine` is `AMM` account.
+ - Adds `tfLPToken`, `tfSingleAsset`, `tfTwoAsset`, `tfOneAssetLPToken`, `tfLimitLPToken`, `tfTwoAssetIfEmpty`,
+ `tfWithdrawAll`, `tfOneAssetWithdrawAll` which allow a trader to specify different fields combination 
+ for `AMMDeposit` and `AMMWithdraw` transactions.
+ - Adds new transaction result codes:
+ - tecUNFUNDED_AMM: insufficient balance to fund AMM. The account does not have funds for liquidity provision.
+ - tecAMM_BALANCE: AMM has invalid balance. Calculated balances greater than the current pool balances.
+ - tecAMM_FAILED: AMM transaction failed. Fails due to a processing failure.
+ - tecAMM_INVALID_TOKENS: AMM invalid LP tokens. Invalid input values, format, or calculated values.
+ - tecAMM_EMPTY: AMM is in empty state. Transaction expects AMM in non-empty state (LP tokens > 0).
+ - tecAMM_NOT_EMPTY: AMM is not in empty state. Transaction expects AMM in empty state (LP tokens == 0).
+ - tecAMM_ACCOUNT: AMM account. Clawback of AMM account.
+ - tecINCOMPLETE: Some work was completed, but more submissions required to finish. AMMDelete partially deletes the trustlines.
 
 ## XRP Ledger version 1.11.0
 

Builds/CMake/RippledCore.cmake

@@ -85,6 +85,7 @@ target_sources (xrpl_core PRIVATE
  src/ripple/protocol/impl/STIssue.cpp
  src/ripple/protocol/impl/Keylet.cpp
  src/ripple/protocol/impl/LedgerFormats.cpp
+ src/ripple/protocol/impl/LedgerHeader.cpp
  src/ripple/protocol/impl/PublicKey.cpp
  src/ripple/protocol/impl/Quality.cpp
  src/ripple/protocol/impl/QualityFunction.cpp
@@ -116,6 +117,9 @@ target_sources (xrpl_core PRIVATE
  src/ripple/protocol/impl/UintTypes.cpp
  src/ripple/protocol/impl/digest.cpp
  src/ripple/protocol/impl/tokens.cpp
+ src/ripple/protocol/impl/NFTSyntheticSerializer.cpp
+ src/ripple/protocol/impl/NFTokenID.cpp
+ src/ripple/protocol/impl/NFTokenOfferID.cpp
  #[===============================[
  main sources:
  subdir: crypto
@@ -195,6 +199,7 @@ install (
  src/ripple/basics/StringUtilities.h
  src/ripple/basics/TaggedCache.h
  src/ripple/basics/tagged_integer.h
+ src/ripple/basics/SubmitSync.h
  src/ripple/basics/ThreadSafetyAnalysis.h
  src/ripple/basics/ToString.h
  src/ripple/basics/UnorderedContainers.h
@@ -232,6 +237,7 @@ install (
  src/ripple/protocol/BuildInfo.h
  src/ripple/protocol/ErrorCodes.h
  src/ripple/protocol/Feature.h
+ src/ripple/protocol/Fees.h
  src/ripple/protocol/HashPrefix.h
  src/ripple/protocol/Indexes.h
  src/ripple/protocol/InnerObjectFormats.h
@@ -240,6 +246,10 @@ install (
  src/ripple/protocol/Keylet.h
  src/ripple/protocol/KnownFormats.h
  src/ripple/protocol/LedgerFormats.h
+ src/ripple/protocol/LedgerHeader.h
+ src/ripple/protocol/NFTSyntheticSerializer.h
+ src/ripple/protocol/NFTokenID.h
+ src/ripple/protocol/NFTokenOfferID.h
  src/ripple/protocol/Protocol.h
  src/ripple/protocol/PublicKey.h
  src/ripple/protocol/Quality.h
@@ -277,6 +287,9 @@ install (
  src/ripple/protocol/UintTypes.h
  src/ripple/protocol/digest.h
  src/ripple/protocol/jss.h
+ src/ripple/protocol/serialize.h
+ src/ripple/protocol/nft.h
+ src/ripple/protocol/nftPageMask.h
  src/ripple/protocol/tokens.h
  DESTINATION include/ripple/protocol)
 install (
@@ -296,6 +309,7 @@ install (
  DESTINATION include/ripple/beast/clock)
 install (
  FILES
+ src/ripple/beast/core/CurrentThreadName.h
  src/ripple/beast/core/LexicalCast.h
  src/ripple/beast/core/List.h
  src/ripple/beast/core/SemanticVersion.h
@@ -309,6 +323,14 @@ install (
 install (
  FILES src/ripple/beast/hash/impl/xxhash.h
  DESTINATION include/ripple/beast/hash/impl)
+install (
+ FILES
+ src/ripple/beast/net/IPAddress.h
+ src/ripple/beast/net/IPAddressConversion.h
+ src/ripple/beast/net/IPAddressV4.h
+ src/ripple/beast/net/IPAddressV6.h
+ src/ripple/beast/net/IPEndpoint.h
+ DESTINATION include/ripple/beast/net)
 install (
  FILES
  src/ripple/beast/rfc2616.h
@@ -476,6 +498,7 @@ target_sources (rippled PRIVATE
  src/ripple/app/rdb/impl/Wallet.cpp
  src/ripple/app/tx/impl/AMMBid.cpp
  src/ripple/app/tx/impl/AMMCreate.cpp
+ src/ripple/app/tx/impl/AMMDelete.cpp
  src/ripple/app/tx/impl/AMMDeposit.cpp
  src/ripple/app/tx/impl/AMMVote.cpp
  src/ripple/app/tx/impl/AMMWithdraw.cpp
@@ -713,9 +736,6 @@ target_sources (rippled PRIVATE
  src/ripple/rpc/impl/ShardVerificationScheduler.cpp
  src/ripple/rpc/impl/Status.cpp
  src/ripple/rpc/impl/TransactionSign.cpp
- src/ripple/rpc/impl/NFTokenID.cpp
- src/ripple/rpc/impl/NFTokenOfferID.cpp
- src/ripple/rpc/impl/NFTSyntheticSerializer.cpp
  #[===============================[
  main sources:
  subdir: perflog
@@ -768,6 +788,7 @@ if (tests)
  src/test/app/HashRouter_test.cpp
  src/test/app/LedgerHistory_test.cpp
  src/test/app/LedgerLoad_test.cpp
+ src/test/app/LedgerMaster_test.cpp
  src/test/app/LedgerReplay_test.cpp
  src/test/app/LoadFeeTrack_test.cpp
  src/test/app/Manifest_test.cpp

Builds/CMake/RippledMultiConfig.cmake

@@ -14,7 +14,7 @@ if (is_multiconfig)
  file(GLOB md_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS
  *.md)
  LIST(APPEND all_sources ${md_files})
- foreach (_target secp256k1::secp256k1 ed25519::ed25519 pbufs xrpl_core rippled)
+ foreach (_target secp256k1::secp256k1 ed25519::ed25519 xrpl_core rippled)
  get_target_property (_type ${_target} TYPE)
  if(_type STREQUAL "INTERFACE_LIBRARY")
  continue()

Builds/CMake/RippledRelease.cmake

@@ -49,7 +49,7 @@ if (is_root_project)
  docker run
  -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled
  -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
- -t rippleci/rippled-rpm-builder:${container_label}
+ -t rippled-rpm-builder:${container_label}
  /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh"
  VERBATIM
  USES_TERMINAL

Builds/CMake/deps/Protobuf.cmake

@@ -1,22 +1,27 @@
 find_package(Protobuf 3.8)
 
-file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/proto_gen)
+set(output_dir ${CMAKE_BINARY_DIR}/proto_gen)
+file(MAKE_DIRECTORY ${output_dir})
 set(ccbd ${CMAKE_CURRENT_BINARY_DIR})
-set(CMAKE_CURRENT_BINARY_DIR ${CMAKE_BINARY_DIR}/proto_gen)
+set(CMAKE_CURRENT_BINARY_DIR ${output_dir})
 protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS src/ripple/proto/ripple.proto)
 set(CMAKE_CURRENT_BINARY_DIR ${ccbd})
 
-add_library(pbufs STATIC ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(pbufs SYSTEM PUBLIC
- ${CMAKE_BINARY_DIR}/proto_gen
- ${CMAKE_BINARY_DIR}/proto_gen/src/ripple/proto
+target_include_directories(xrpl_core SYSTEM PUBLIC
+ # The generated implementation imports the header relative to the output
+ # directory.
+ $<BUILD_INTERFACE:${output_dir}>
+ $<BUILD_INTERFACE:${output_dir}/src>
 )
-target_link_libraries(pbufs protobuf::libprotobuf)
-target_compile_options(pbufs
+target_sources(xrpl_core PRIVATE ${output_dir}/src/ripple/proto/ripple.pb.cc)
+install(
+ FILES ${output_dir}/src/ripple/proto/ripple.pb.h
+ DESTINATION include/ripple/proto)
+target_link_libraries(xrpl_core PUBLIC protobuf::libprotobuf)
+target_compile_options(xrpl_core
  PUBLIC
  $<$<BOOL:${XCODE}>:
  --system-header-prefix="google/protobuf"
  -Wno-deprecated-dynamic-exception-spec
  >
 )
-add_library(Ripple::pbufs ALIAS pbufs)

Builds/CMake/deps/gRPC.cmake

@@ -5,21 +5,30 @@ find_package(gRPC 1.23)
  grpc defs and bundle into a
  static lib
 #]=================================]
-set(GRPC_GEN_DIR "${CMAKE_BINARY_DIR}/proto_gen_grpc")
+set(output_dir "${CMAKE_BINARY_DIR}/proto_gen_grpc")
+set(GRPC_GEN_DIR "${output_dir}/ripple/proto")
 file(MAKE_DIRECTORY ${GRPC_GEN_DIR})
 set(GRPC_PROTO_SRCS)
 set(GRPC_PROTO_HDRS)
 set(GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org")
-file(GLOB_RECURSE GRPC_DEFINITION_FILES LIST_DIRECTORIES false "${GRPC_PROTO_ROOT}/*.proto")
+file(GLOB_RECURSE GRPC_DEFINITION_FILES "${GRPC_PROTO_ROOT}/*.proto")
 foreach(file ${GRPC_DEFINITION_FILES})
+ # /home/user/rippled/src/ripple/proto/org/.../v1/get_ledger.proto
  get_filename_component(_abs_file ${file} ABSOLUTE)
+ # /home/user/rippled/src/ripple/proto/org/.../v1
  get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
+ # get_ledger
  get_filename_component(_basename ${file} NAME_WE)
+ # /home/user/rippled/src/ripple/proto
  get_filename_component(_proto_inc ${GRPC_PROTO_ROOT} DIRECTORY) # updir one level
+ # org/.../v1/get_ledger.proto
  file(RELATIVE_PATH _rel_root_file ${_proto_inc} ${_abs_file})
+ # org/.../v1
  get_filename_component(_rel_root_dir ${_rel_root_file} DIRECTORY)
+ # src/ripple/proto/org/.../v1
  file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
 
+ # .cmake/proto_gen_grpc/ripple/proto/org/.../v1/get_ledger.grpc.pb.cc
  set(src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc")
  set(src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc")
  set(hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h")
@@ -36,20 +45,32 @@ foreach(file ${GRPC_DEFINITION_FILES})
  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMENT "Running gRPC C++ protocol buffer compiler on ${file}"
  VERBATIM)
- set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES GENERATED TRUE)
+ set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES
+ GENERATED TRUE
+ SKIP_UNITY_BUILD_INCLUSION ON
+ )
  list(APPEND GRPC_PROTO_SRCS ${src_1} ${src_2})
  list(APPEND GRPC_PROTO_HDRS ${hdr_1} ${hdr_2})
 endforeach()
 
-add_library(grpc_pbufs STATIC ${GRPC_PROTO_SRCS} ${GRPC_PROTO_HDRS})
-#target_include_directories(grpc_pbufs PRIVATE src)
-target_include_directories(grpc_pbufs SYSTEM PUBLIC ${GRPC_GEN_DIR})
-target_link_libraries(grpc_pbufs
+target_include_directories(xrpl_core SYSTEM PUBLIC
+ $<BUILD_INTERFACE:${output_dir}>
+ $<BUILD_INTERFACE:${output_dir}/ripple/proto>
+ # The generated sources include headers relative to this path. Fix it later.
+ $<INSTALL_INTERFACE:include/ripple/proto>
+)
+target_sources(xrpl_core PRIVATE ${GRPC_PROTO_SRCS})
+install(
+ DIRECTORY ${output_dir}/ripple
+ DESTINATION include/
+ FILES_MATCHING PATTERN "*.h"
+)
+target_link_libraries(xrpl_core PUBLIC
  "gRPC::grpc++"
  # libgrpc is missing references.
  absl::random_random
 )
-target_compile_options(grpc_pbufs
+target_compile_options(xrpl_core
  PRIVATE
  $<$<BOOL:${MSVC}>:-wd4065>
  $<$<NOT:$<BOOL:${MSVC}>>:-Wno-deprecated-declarations>
@@ -59,4 +80,3 @@ target_compile_options(grpc_pbufs
  --system-header-prefix="google/protobuf"
  -Wno-deprecated-dynamic-exception-spec
  >)
-add_library(Ripple::grpc_pbufs ALIAS grpc_pbufs)

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -14,7 +14,7 @@ variables:
  RPM_CONTAINER_TAG: "2023-02-13"
  RPM_CONTAINER_NAME: "rippled-rpm-builder"
  RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
- DPKG_CONTAINER_TAG: "2023-03-20"
+ DPKG_CONTAINER_TAG: "2023-07-31"
  DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
  DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
  ARTIFACTORY_HOST: "artifactory.ops.ripple.com"

Builds/containers/packaging/dpkg/debian/rippled-reporting.install

@@ -1,8 +1,8 @@
-bld/rippled-reporting/rippled-reporting opt/rippled-reporting/bin
+build.rippled-reporting/rippled-reporting opt/rippled-reporting/bin
 cfg/rippled-reporting.cfg opt/rippled-reporting/etc
 debian/tmp/opt/rippled-reporting/etc/validators.txt opt/rippled-reporting/etc
 
 opt/rippled-reporting/bin/update-rippled-reporting.sh
 opt/rippled-reporting/bin/getRippledReportingInfo
 opt/rippled-reporting/etc/update-rippled-reporting-cron
-etc/logrotate.d/rippled-reporting
+etc/logrotate.d/rippled-reporting

Builds/containers/packaging/dpkg/debian/rules

@@ -15,69 +15,56 @@ override_dh_systemd_start:
  dh_systemd_start --no-restart-on-upgrade
 
 override_dh_auto_configure:
- apt install --yes gcc-11 g++-11
- update-alternatives --install \
- /usr/bin/gcc gcc /usr/bin/gcc-11 100 \
- --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
- --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
- --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \
- --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \
- --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
- --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \
- --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11
- update-alternatives --set gcc /usr/bin/gcc-11
- env
- rm -rf bld
- conan profile update settings.compiler.cppstd=20 gcc
- conan profile update settings.compiler.version=11 gcc
- conan export external/snappy snappy/1.1.9@
+ /root/.pyenv/shims/conan export external/snappy snappy/1.1.10@
+ /root/.pyenv/shims/conan export external/soci soci/4.0.3@
 
- conan install . \
- --profile gcc \
- --install-folder bld/rippled \
- --build missing \
- --build boost \
- --build sqlite3 \
- --settings build_type=Release
+ mkdir build.rippled
 
- cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
- -G Ninja \
- -DCMAKE_BUILD_TYPE=Release \
- -DCMAKE_INSTALL_PREFIX=/opt/ripple \
- -Dstatic=ON \
- -Dunity=OFF \
- -DCMAKE_VERBOSE_MAKEFILE=ON \
- -Dvalidator_keys=ON \
- -B bld/rippled
+ cd build.rippled && \
+ /root/.pyenv/shims/conan install .. \
+ --profile gcc \
+ --output-folder . \
+ --build missing \
+ --settings build_type=Release
 
- conan install . \
- --profile gcc \
- --install-folder bld/rippled-reporting \
- --build missing \
- --build boost \
- --build sqlite3 \
- --build libuv \
- --settings build_type=Release \
- --settings compiler.cppstd=17 \
- --options reporting=True
+ cd build.rippled && \
+ cmake .. \
+ -DCMAKE_BUILD_TYPE=Release \
+ -Dvalidator_keys=ON \
+ -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake
 
- cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
- -G Ninja \
- -DCMAKE_BUILD_TYPE=Release \
- -DCMAKE_INSTALL_PREFIX=/opt/rippled-reporting \
- -Dstatic=ON \
- -Dunity=OFF \
- -DCMAKE_VERBOSE_MAKEFILE=ON \
- -Dreporting=ON \
- -B bld/rippled-reporting
+ mkdir build.rippled-reporting
+
+ cd build.rippled-reporting && \
+ /root/.pyenv/shims/conan install .. \
+ --profile gcc \
+ --output-folder . \
+ --settings compiler.cppstd=17 \
+ --settings build_type=Release \
+ --build missing \
+ --build boost \
+ --build sqlite3 \
+ --build libuv \
+ --options reporting=True
+
+ cd build.rippled-reporting && \
+ cmake .. \
+ -DCMAKE_BUILD_TYPE=Release \
+ -Dvalidator_keys=ON \
+ -Dstatic=ON \
+ -Dunity=OFF \
+ -Dreporting=ON \
+ -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake
 
 override_dh_auto_build:
- cmake --build bld/rippled --target rippled --target validator-keys -j${nproc}
- cmake --build bld/rippled-reporting --target rippled -j${nproc}
+ cmake --build build.rippled --target rippled --target validator-keys --parallel 8
+ cmake --build build.rippled-reporting --target rippled --parallel 8
 
 override_dh_auto_install:
- cmake --install bld/rippled --prefix debian/tmp/opt/ripple
- install -D bld/rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
+ cmake --install build.rippled --prefix debian/tmp/opt/ripple
+ install -D build.rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
  install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
  install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo
  install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron