Fix the package recipe for consumers of libxrpl (XRPLF#4631)

- "Rename" the type `LedgerInfo` to `LedgerHeader` (but leave an alias for `LedgerInfo` to not yet disturb existing uses). Put it in its own public header, named after itself, so that it is more easily found. - Move the type `Fees` and NFT serialization functions into public (installed) headers. - Compile the XRPL and gRPC protocol buffers directly into `libxrpl` and install their headers. Fix the Conan recipe to correctly export these types. Addresses change (2) in XRPLF/XRPL-Standards#121. For context: This work supports Clio's dependence on libxrpl. Clio is just an example consumer. These changes should benefit all current and future consumers. --------- Co-authored-by: cyan317 <[email protected]>
manojsdoshi · Aug 18, 2023 · bd35b4f · bd35b4f
1 parent 0b674bb
commit bd35b4f
Show file tree

Hide file tree

Showing 38 changed files with 528 additions and 381 deletions.
diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake
@@ -85,6 +85,7 @@ target_sources (xrpl_core PRIVATE
  src/ripple/protocol/impl/STIssue.cpp
  src/ripple/protocol/impl/Keylet.cpp
  src/ripple/protocol/impl/LedgerFormats.cpp
+ src/ripple/protocol/impl/LedgerHeader.cpp
  src/ripple/protocol/impl/PublicKey.cpp
  src/ripple/protocol/impl/Quality.cpp
  src/ripple/protocol/impl/QualityFunction.cpp
@@ -116,6 +117,9 @@ target_sources (xrpl_core PRIVATE
  src/ripple/protocol/impl/UintTypes.cpp
  src/ripple/protocol/impl/digest.cpp
  src/ripple/protocol/impl/tokens.cpp
+ src/ripple/protocol/impl/NFTSyntheticSerializer.cpp
+ src/ripple/protocol/impl/NFTokenID.cpp
+ src/ripple/protocol/impl/NFTokenOfferID.cpp
  #[===============================[
  main sources:
  subdir: crypto
@@ -232,6 +236,7 @@ install (
  src/ripple/protocol/BuildInfo.h
  src/ripple/protocol/ErrorCodes.h
  src/ripple/protocol/Feature.h
+ src/ripple/protocol/Fees.h
  src/ripple/protocol/HashPrefix.h
  src/ripple/protocol/Indexes.h
  src/ripple/protocol/InnerObjectFormats.h
@@ -240,6 +245,10 @@ install (
  src/ripple/protocol/Keylet.h
  src/ripple/protocol/KnownFormats.h
  src/ripple/protocol/LedgerFormats.h
+ src/ripple/protocol/LedgerHeader.h
+ src/ripple/protocol/NFTSyntheticSerializer.h
+ src/ripple/protocol/NFTokenID.h
+ src/ripple/protocol/NFTokenOfferID.h
  src/ripple/protocol/Protocol.h
  src/ripple/protocol/PublicKey.h
  src/ripple/protocol/Quality.h
@@ -277,6 +286,9 @@ install (
  src/ripple/protocol/UintTypes.h
  src/ripple/protocol/digest.h
  src/ripple/protocol/jss.h
+ src/ripple/protocol/serialize.h
+ src/ripple/protocol/nft.h
+ src/ripple/protocol/nftPageMask.h
  src/ripple/protocol/tokens.h
  DESTINATION include/ripple/protocol)
 install (
@@ -296,6 +308,7 @@ install (
  DESTINATION include/ripple/beast/clock)
 install (
  FILES
+ src/ripple/beast/core/CurrentThreadName.h
  src/ripple/beast/core/LexicalCast.h
  src/ripple/beast/core/List.h
  src/ripple/beast/core/SemanticVersion.h
@@ -309,6 +322,14 @@ install (
 install (
  FILES src/ripple/beast/hash/impl/xxhash.h
  DESTINATION include/ripple/beast/hash/impl)
+install (
+ FILES
+ src/ripple/beast/net/IPAddress.h
+ src/ripple/beast/net/IPAddressConversion.h
+ src/ripple/beast/net/IPAddressV4.h
+ src/ripple/beast/net/IPAddressV6.h
+ src/ripple/beast/net/IPEndpoint.h
+ DESTINATION include/ripple/beast/net)
 install (
  FILES
  src/ripple/beast/rfc2616.h
@@ -714,9 +735,6 @@ target_sources (rippled PRIVATE
  src/ripple/rpc/impl/ShardVerificationScheduler.cpp
  src/ripple/rpc/impl/Status.cpp
  src/ripple/rpc/impl/TransactionSign.cpp
- src/ripple/rpc/impl/NFTokenID.cpp
- src/ripple/rpc/impl/NFTokenOfferID.cpp
- src/ripple/rpc/impl/NFTSyntheticSerializer.cpp
  #[===============================[
  main sources:
  subdir: perflog

diff --git a/Builds/CMake/RippledMultiConfig.cmake b/Builds/CMake/RippledMultiConfig.cmake
@@ -14,7 +14,7 @@ if (is_multiconfig)
  file(GLOB md_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS
  *.md)
  LIST(APPEND all_sources ${md_files})
- foreach (_target secp256k1::secp256k1 ed25519::ed25519 pbufs xrpl_core rippled)
+ foreach (_target secp256k1::secp256k1 ed25519::ed25519 xrpl_core rippled)
  get_target_property (_type ${_target} TYPE)
  if(_type STREQUAL "INTERFACE_LIBRARY")
  continue()

diff --git a/Builds/CMake/deps/Protobuf.cmake b/Builds/CMake/deps/Protobuf.cmake
@@ -1,22 +1,27 @@
 find_package(Protobuf 3.8)
 
-file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/proto_gen)
+set(output_dir ${CMAKE_BINARY_DIR}/proto_gen)
+file(MAKE_DIRECTORY ${output_dir})
 set(ccbd ${CMAKE_CURRENT_BINARY_DIR})
-set(CMAKE_CURRENT_BINARY_DIR ${CMAKE_BINARY_DIR}/proto_gen)
+set(CMAKE_CURRENT_BINARY_DIR ${output_dir})
 protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS src/ripple/proto/ripple.proto)
 set(CMAKE_CURRENT_BINARY_DIR ${ccbd})
 
-add_library(pbufs STATIC ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(pbufs SYSTEM PUBLIC
- ${CMAKE_BINARY_DIR}/proto_gen
- ${CMAKE_BINARY_DIR}/proto_gen/src/ripple/proto
+target_include_directories(xrpl_core SYSTEM PUBLIC
+ # The generated implementation imports the header relative to the output
+ # directory.
+ $<BUILD_INTERFACE:${output_dir}>
+ $<BUILD_INTERFACE:${output_dir}/src>
 )
-target_link_libraries(pbufs protobuf::libprotobuf)
-target_compile_options(pbufs
+target_sources(xrpl_core PRIVATE ${output_dir}/src/ripple/proto/ripple.pb.cc)
+install(
+ FILES ${output_dir}/src/ripple/proto/ripple.pb.h
+ DESTINATION include/ripple/proto)
+target_link_libraries(xrpl_core PUBLIC protobuf::libprotobuf)
+target_compile_options(xrpl_core
  PUBLIC
  $<$<BOOL:${XCODE}>:
  --system-header-prefix="google/protobuf"
  -Wno-deprecated-dynamic-exception-spec
  >
 )
-add_library(Ripple::pbufs ALIAS pbufs)
diff --git a/Builds/CMake/deps/gRPC.cmake b/Builds/CMake/deps/gRPC.cmake
@@ -5,21 +5,30 @@ find_package(gRPC 1.23)
  grpc defs and bundle into a
  static lib
 #]=================================]
-set(GRPC_GEN_DIR "${CMAKE_BINARY_DIR}/proto_gen_grpc")
+set(output_dir "${CMAKE_BINARY_DIR}/proto_gen_grpc")
+set(GRPC_GEN_DIR "${output_dir}/ripple/proto")
 file(MAKE_DIRECTORY ${GRPC_GEN_DIR})
 set(GRPC_PROTO_SRCS)
 set(GRPC_PROTO_HDRS)
 set(GRPC_PROTO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src/ripple/proto/org")
-file(GLOB_RECURSE GRPC_DEFINITION_FILES LIST_DIRECTORIES false "${GRPC_PROTO_ROOT}/*.proto")
+file(GLOB_RECURSE GRPC_DEFINITION_FILES "${GRPC_PROTO_ROOT}/*.proto")
 foreach(file ${GRPC_DEFINITION_FILES})
+ # /home/user/rippled/src/ripple/proto/org/.../v1/get_ledger.proto
  get_filename_component(_abs_file ${file} ABSOLUTE)
+ # /home/user/rippled/src/ripple/proto/org/.../v1
  get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
+ # get_ledger
  get_filename_component(_basename ${file} NAME_WE)
+ # /home/user/rippled/src/ripple/proto
  get_filename_component(_proto_inc ${GRPC_PROTO_ROOT} DIRECTORY) # updir one level
+ # org/.../v1/get_ledger.proto
  file(RELATIVE_PATH _rel_root_file ${_proto_inc} ${_abs_file})
+ # org/.../v1
  get_filename_component(_rel_root_dir ${_rel_root_file} DIRECTORY)
+ # src/ripple/proto/org/.../v1
  file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
 
+ # .cmake/proto_gen_grpc/ripple/proto/org/.../v1/get_ledger.grpc.pb.cc
  set(src_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.cc")
  set(src_2 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.pb.cc")
  set(hdr_1 "${GRPC_GEN_DIR}/${_rel_root_dir}/${_basename}.grpc.pb.h")
@@ -36,20 +45,32 @@ foreach(file ${GRPC_DEFINITION_FILES})
  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMENT "Running gRPC C++ protocol buffer compiler on ${file}"
  VERBATIM)
- set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES GENERATED TRUE)
+ set_source_files_properties(${src_1} ${src_2} ${hdr_1} ${hdr_2} PROPERTIES
+ GENERATED TRUE
+ SKIP_UNITY_BUILD_INCLUSION ON
+ )
  list(APPEND GRPC_PROTO_SRCS ${src_1} ${src_2})
  list(APPEND GRPC_PROTO_HDRS ${hdr_1} ${hdr_2})
 endforeach()
 
-add_library(grpc_pbufs STATIC ${GRPC_PROTO_SRCS} ${GRPC_PROTO_HDRS})
-#target_include_directories(grpc_pbufs PRIVATE src)
-target_include_directories(grpc_pbufs SYSTEM PUBLIC ${GRPC_GEN_DIR})
-target_link_libraries(grpc_pbufs
+target_include_directories(xrpl_core SYSTEM PUBLIC
+ $<BUILD_INTERFACE:${output_dir}>
+ $<BUILD_INTERFACE:${output_dir}/ripple/proto>
+ # The generated sources include headers relative to this path. Fix it later.
+ $<INSTALL_INTERFACE:include/ripple/proto>
+)
+target_sources(xrpl_core PRIVATE ${GRPC_PROTO_SRCS})
+install(
+ DIRECTORY ${output_dir}/ripple
+ DESTINATION include/
+ FILES_MATCHING PATTERN "*.h"
+)
+target_link_libraries(xrpl_core PUBLIC
  "gRPC::grpc++"
  # libgrpc is missing references.
  absl::random_random
 )
-target_compile_options(grpc_pbufs
+target_compile_options(xrpl_core
  PRIVATE
  $<$<BOOL:${MSVC}>:-wd4065>
  $<$<NOT:$<BOOL:${MSVC}>>:-Wno-deprecated-declarations>
@@ -59,4 +80,3 @@ target_compile_options(grpc_pbufs
  --system-header-prefix="google/protobuf"
  -Wno-deprecated-dynamic-exception-spec
  >)
-add_library(Ripple::grpc_pbufs ALIAS grpc_pbufs)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -93,17 +93,13 @@ endif()
 
 find_package(nudb REQUIRED)
 find_package(date REQUIRED)
-include(deps/Protobuf)
-include(deps/gRPC)
 
 target_link_libraries(ripple_libs INTERFACE
  ed25519::ed25519
  LibArchive::LibArchive
  lz4::lz4
  OpenSSL::Crypto
  OpenSSL::SSL
- Ripple::grpc_pbufs
- Ripple::pbufs
  secp256k1::secp256k1
  soci::soci
  SQLite::SQLite3
@@ -131,6 +127,8 @@ endif()
 ###
 
 include(RippledCore)
+include(deps/Protobuf)
+include(deps/gRPC)
 include(RippledInstall)
 include(RippledCov)
 include(RippledMultiConfig)

diff --git a/conanfile.py b/conanfile.py
@@ -151,9 +151,12 @@ def package_info(self):
  'libed25519.a',
  'libsecp256k1.a',
  ]
- libxrpl.includedirs = ['include']
+ # TODO: Fix the protobufs to include each other relative to
+ # `include/`, not `include/ripple/proto/`.
+ libxrpl.includedirs = ['include', 'include/ripple/proto']
  libxrpl.requires = [
  'boost::boost',
  'openssl::crypto',
  'date::date',
+ 'grpc::grpc++',
  ]
diff --git a/src/ripple/app/ledger/InboundLedger.h b/src/ripple/app/ledger/InboundLedger.h
@@ -197,14 +197,6 @@ class InboundLedger final : public TimeoutCounter,
  std::unique_ptr<PeerSet> mPeerSet;
 };
 
-/** Deserialize a ledger header from a byte array. */
-LedgerInfo
-deserializeHeader(Slice data, bool hasHash = false);
-
-/** Deserialize a ledger header (prefixed with 4 bytes) from a byte array. */
-LedgerInfo
-deserializePrefixedHeader(Slice data, bool hasHash = false);
-
 } // namespace ripple
 
 #endif
diff --git a/src/ripple/app/ledger/LedgerToJson.h b/src/ripple/app/ledger/LedgerToJson.h
@@ -26,6 +26,7 @@
 #include <ripple/json/Object.h>
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/jss.h>
+#include <ripple/protocol/serialize.h>
 #include <ripple/rpc/Context.h>
 
 namespace ripple {
@@ -70,22 +71,6 @@ addJson(Json::Value&, LedgerFill const&);
 Json::Value
 getJson(LedgerFill const&);
 
-/** Serialize an object to a blob. */
-template <class Object>
-Blob
-serializeBlob(Object const& o)
-{
- Serializer s;
- o.add(s);
- return s.peekData();
-}
-
-/** Serialize an object to a hex string. */
-inline std::string
-serializeHex(STObject const& o)
-{
- return strHex(serializeBlob(o));
-}
 } // namespace ripple
 
 #endif
diff --git a/src/ripple/app/ledger/impl/InboundLedger.cpp b/src/ripple/app/ledger/impl/InboundLedger.cpp
@@ -271,36 +271,6 @@ InboundLedger::neededStateHashes(int max, SHAMapSyncFilter* filter) const
  mLedger->info().accountHash, mLedger->stateMap(), max, filter);
 }
 
-LedgerInfo
-deserializeHeader(Slice data, bool hasHash)
-{
- SerialIter sit(data.data(), data.size());
-
- LedgerInfo info;
-
- info.seq = sit.get32();
- info.drops = sit.get64();
- info.parentHash = sit.get256();
- info.txHash = sit.get256();
- info.accountHash = sit.get256();
- info.parentCloseTime =
- NetClock::time_point{NetClock::duration{sit.get32()}};
- info.closeTime = NetClock::time_point{NetClock::duration{sit.get32()}};
- info.closeTimeResolution = NetClock::duration{sit.get8()};
- info.closeFlags = sit.get8();
-
- if (hasHash)
- info.hash = sit.get256();
-
- return info;
-}
-
-LedgerInfo
-deserializePrefixedHeader(Slice data, bool hasHash)
-{
- return deserializeHeader(data + 4, hasHash);
-}
-
 // See how much of the ledger data is stored locally
 // Data found in a fetch pack will be stored
 void
@@ -568,10 +538,9 @@ InboundLedger::trigger(std::shared_ptr<Peer> const& peer, TriggerReason reason)
 
  if (auto stream = journal_.trace())
  {
+ stream << "Trigger acquiring ledger " << hash_;
  if (peer)
- stream << "Trigger acquiring ledger " << hash_ << " from " << peer;
- else
- stream << "Trigger acquiring ledger " << hash_;
+ stream << " from " << peer;
 
  if (complete_ || failed_)
  stream << "complete=" << complete_ << " failed=" << failed_;

diff --git a/src/ripple/app/ledger/impl/LedgerReplayMsgHandler.cpp b/src/ripple/app/ledger/impl/LedgerReplayMsgHandler.cpp
@@ -21,6 +21,7 @@
 #include <ripple/app/ledger/LedgerReplayer.h>
 #include <ripple/app/ledger/impl/LedgerReplayMsgHandler.h>
 #include <ripple/app/main/Application.h>
+#include <ripple/protocol/LedgerHeader.h>
 
 #include <memory>
 

diff --git a/src/ripple/app/reporting/ETLSource.h b/src/ripple/app/reporting/ETLSource.h
@@ -21,6 +21,7 @@
 #define RIPPLE_APP_REPORTING_ETLSOURCE_H_INCLUDED
 #include <ripple/app/main/Application.h>
 #include <ripple/app/reporting/ETLHelpers.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 #include <ripple/protocol/STLedgerEntry.h>
 #include <ripple/rpc/Context.h>
 
@@ -29,7 +30,6 @@
 #include <boost/beast/core/string.hpp>
 #include <boost/beast/websocket.hpp>
 
-#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
 #include <grpcpp/grpcpp.h>
 
 namespace ripple {

diff --git a/src/ripple/app/reporting/P2pProxy.h b/src/ripple/app/reporting/P2pProxy.h
@@ -21,12 +21,11 @@
 #define RIPPLE_APP_REPORTING_P2PPROXY_H_INCLUDED
 
 #include <ripple/app/main/Application.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/impl/Handler.h>
 
 #include <boost/beast/websocket.hpp>
-
-#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
 #include <grpcpp/grpcpp.h>
 
 namespace ripple {