Enforce levelization in libxrpl with CMake

Builds/levelization/results/loops.txt

@@ -4,9 +4,6 @@ Loop: test.jtx test.toplevel
 Loop: test.jtx test.unit_test
  test.unit_test == test.jtx
 
-Loop: xrpl.basics xrpl.json
- xrpl.json ~= xrpl.basics
-
 Loop: xrpld.app xrpld.core
  xrpld.app > xrpld.core
 

Builds/levelization/results/ordering.txt

@@ -1,5 +1,4 @@
 libxrpl.basics > xrpl.basics
-libxrpl.basics > xrpl.protocol
 libxrpl.crypto > xrpl.basics
 libxrpl.json > xrpl.basics
 libxrpl.json > xrpl.json
@@ -130,6 +129,7 @@ test.shamap > xrpl.protocol
 test.toplevel > test.csf
 test.toplevel > xrpl.json
 test.unit_test > xrpl.basics
+xrpl.json > xrpl.basics
 xrpl.protocol > xrpl.basics
 xrpl.protocol > xrpl.json
 xrpl.resource > xrpl.basics

cmake/RippledCore.cmake

@@ -47,6 +47,57 @@ target_link_libraries(xrpl.libpb
  gRPC::grpc++
 )
 
+# TODO: Clean up the number of library targets later.
+add_library(xrpl.imports.main INTERFACE)
+target_link_libraries(xrpl.imports.main INTERFACE
+ LibArchive::LibArchive
+ OpenSSL::Crypto
+ Ripple::boost
+ Ripple::opts
+ Ripple::syslibs
+ absl::random_random
+ date::date
+ ed25519::ed25519
+ secp256k1::secp256k1
+ xxHash::xxhash
+)
+
+include(add_module)
+include(target_link_modules)
+
+# Level 01
+add_module(xrpl beast)
+target_link_libraries(xrpl.libxrpl.beast PUBLIC
+ xrpl.imports.main
+ xrpl.libpb
+)
+
+# Level 02
+add_module(xrpl basics)
+target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast)
+
+# Level 03
+add_module(xrpl json)
+target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics)
+
+add_module(xrpl crypto)
+target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics)
+
+# Level 04
+add_module(xrpl protocol)
+target_link_libraries(xrpl.libxrpl.protocol PUBLIC
+ xrpl.libxrpl.crypto
+ xrpl.libxrpl.json
+)
+
+# Level 05
+add_module(xrpl resource)
+target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol)
+
+add_module(xrpl server)
+target_link_libraries(xrpl.libxrpl.server PUBLIC xrpl.libxrpl.protocol)
+
+
 add_library(xrpl.libxrpl)
 set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl)
 if(unity)
@@ -60,10 +111,24 @@ file(GLOB_RECURSE sources CONFIGURE_DEPENDS
 )
 target_sources(xrpl.libxrpl PRIVATE ${sources})
 
-target_include_directories(xrpl.libxrpl
- PUBLIC
- $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
- $<INSTALL_INTERFACE:include>)
+target_link_modules(xrpl PUBLIC
+ basics
+ beast
+ crypto
+ json
+ protocol
+ resource
+ server
+)
+
+# All headers in libxrpl are in modules.
+# Uncomment this stanza if you have not yet moved new headers into a module.
+# target_include_directories(xrpl.libxrpl
+# PRIVATE
+# $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+# PUBLIC
+# $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+# $<INSTALL_INTERFACE:include>)
 
 target_compile_definitions(xrpl.libxrpl
  PUBLIC

cmake/RippledInstall.cmake

@@ -8,7 +8,15 @@ install (
  opts
  ripple_syslibs
  ripple_boost
+ xrpl.imports.main
  xrpl.libpb
+ xrpl.libxrpl.basics
+ xrpl.libxrpl.beast
+ xrpl.libxrpl.crypto
+ xrpl.libxrpl.json
+ xrpl.libxrpl.protocol
+ xrpl.libxrpl.resource
+ xrpl.libxrpl.server
  xrpl.libxrpl
  EXPORT RippleExports
  LIBRARY DESTINATION lib

cmake/add_module.cmake

@@ -0,0 +1,37 @@
+include(isolate_headers)
+
+# Create an OBJECT library target named
+#
+# ${PROJECT_NAME}.lib${parent}.${name}
+#
+# with sources in src/lib${parent}/${name}
+# and headers in include/${parent}/${name}
+# that cannot include headers from other directories in include/
+# unless they come through linked libraries.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+function(add_module parent name)
+ set(target ${PROJECT_NAME}.lib${parent}.${name})
+ add_library(${target} OBJECT)
+ file(GLOB_RECURSE sources CONFIGURE_DEPENDS
+ "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}/*.cpp"
+ )
+ target_sources(${target} PRIVATE ${sources})
+ target_include_directories(${target} PUBLIC
+ "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+ )
+ isolate_headers(
+ ${target}
+ "${CMAKE_CURRENT_SOURCE_DIR}/include"
+ "${CMAKE_CURRENT_SOURCE_DIR}/include/${parent}/${name}"
+ PUBLIC
+ )
+ isolate_headers(
+ ${target}
+ "${CMAKE_CURRENT_SOURCE_DIR}/src"
+ "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}"
+ PRIVATE
+ )
+endfunction()

cmake/isolate_headers.cmake

@@ -0,0 +1,46 @@
+# Consider include directory B nested under prefix A:
+#
+# /path/to/A/then/to/B/...
+#
+# Call C the relative path from A to B.
+# C is what we want to write in `#include` directives:
+#
+# #include <then/to/B/...>
+#
+# Examples, all from the `jobqueue` module:
+#
+# - Library public headers:
+# B = /include/xrpl/jobqueue
+# A = /include/
+# C = xrpl/jobqueue
+#
+# - Library private headers:
+# B = /src/libxrpl/jobqueue
+# A = /src/
+# C = libxrpl/jobqueue
+#
+# - Test private headers:
+# B = /tests/jobqueue
+# A = /
+# C = tests/jobqueue
+#
+# To isolate headers from each other,
+# we want to create a symlink Y that points to B,
+# within a subdirectory X of the `CMAKE_BINARY_DIR`,
+# that has the same relative path C between X and Y,
+# and then add X as an include directory of the target,
+# sometimes `PUBLIC` and sometimes `PRIVATE`.
+# The Cs are all guaranteed to be unique.
+# We can guarantee a unique X per target by using
+# `${CMAKE_CURRENT_BINARY_DIR}/include/${target}`.
+#
+# isolate_headers(target A B scope)
+function(isolate_headers target A B scope)
+ file(RELATIVE_PATH C "${A}" "${B}")
+ set(X "${CMAKE_CURRENT_BINARY_DIR}/modules/${target}")
+ set(Y "${X}/${C}")
+ cmake_path(GET Y PARENT_PATH parent)
+ file(MAKE_DIRECTORY "${parent}")
+ file(CREATE_LINK "${B}" "${Y}" SYMBOLIC)
+ target_include_directories(${target} ${scope} "$<BUILD_INTERFACE:${X}>")
+endfunction()

cmake/target_link_modules.cmake

@@ -0,0 +1,24 @@
+# Link a library to its modules (see: `add_module`)
+# and remove the module sources from the library's sources.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+# add_library(project.libparent)
+# target_link_modules(parent PUBLIC a b)
+function(target_link_modules parent scope)
+ set(library ${PROJECT_NAME}.lib${parent})
+ foreach(name ${ARGN})
+ set(module ${library}.${name})
+ get_target_property(sources ${library} SOURCES)
+ list(LENGTH sources before)
+ get_target_property(dupes ${module} SOURCES)
+ list(LENGTH dupes expected)
+ list(REMOVE_ITEM sources ${dupes})
+ list(LENGTH sources after)
+ math(EXPR actual "${before} - ${after}")
+ message(STATUS "${module} with ${expected} sources took ${actual} sources from ${library}")
+ set_target_properties(${library} PROPERTIES SOURCES "${sources}")
+ target_link_libraries(${library} ${scope} ${module})
+ endforeach()
+endfunction()

include/xrpl/basics/Number.h

@@ -20,7 +20,6 @@
 #ifndef RIPPLE_BASICS_NUMBER_H_INCLUDED
 #define RIPPLE_BASICS_NUMBER_H_INCLUDED
 
-#include <xrpl/basics/XRPAmount.h>
 #include <cstdint>
 #include <limits>
 #include <ostream>
@@ -51,8 +50,6 @@ class Number
  explicit Number(rep mantissa, int exponent);
  explicit constexpr Number(rep mantissa, int exponent, unchecked) noexcept;
 
- Number(XRPAmount const& x);
-
  constexpr rep
  mantissa() const noexcept;
  constexpr int
@@ -88,8 +85,6 @@ class Number
  static constexpr Number
  lowest() noexcept;
 
- explicit
- operator XRPAmount() const; // round to nearest, even on tie
  explicit
  operator rep() const; // round to nearest, even on tie
 
@@ -208,10 +203,6 @@ inline Number::Number(rep mantissa) : Number{mantissa, 0}
 {
 }
 
-inline Number::Number(XRPAmount const& x) : Number{x.drops()}
-{
-}
-
 inline constexpr Number::rep
 Number::mantissa() const noexcept
 {

include/xrpl/basics/SHAMapHash.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED
 
 #include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/partitioned_unordered_map.h>
 
 #include <ostream>
 
@@ -108,6 +109,13 @@
  return !(x == y);
 }
 
+template <>
+inline std::size_t
+extract(SHAMapHash const& key)
+{
+ return *reinterpret_cast<std::size_t const*>(key.as_uint256().data());
+}
+
 } // namespace ripple
 
 #endif // RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED

include/xrpl/basics/base_uint.h

@@ -29,6 +29,7 @@
 #include <xrpl/basics/Slice.h>
 #include <xrpl/basics/contract.h>
 #include <xrpl/basics/hardened_hash.h>
+#include <xrpl/basics/partitioned_unordered_map.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/Zero.h>
 #include <boost/endian/conversion.hpp>
@@ -631,6 +632,17 @@ operator<<(std::ostream& out, base_uint<Bits, Tag> const& u)
  return out << to_string(u);
 }
 
+template <>
+inline std::size_t
+extract(uint256 const& key)
+{
+ std::size_t result;
+ // Use memcpy to avoid unaligned UB
+ // (will optimize to equivalent code)
+ std::memcpy(&result, key.data(), sizeof(std::size_t));
+ return result;
+}
+
 #ifndef __INTELLISENSE__
 static_assert(sizeof(uint128) == 128 / 8, "There should be no padding bytes");
 static_assert(sizeof(uint160) == 160 / 8, "There should be no padding bytes");

include/xrpl/basics/partitioned_unordered_map.h

@@ -20,9 +20,12 @@
 #ifndef RIPPLE_BASICS_PARTITIONED_UNORDERED_MAP_H
 #define RIPPLE_BASICS_PARTITIONED_UNORDERED_MAP_H
 
+#include <xrpl/beast/hash/uhash.h>
+
 #include <cassert>
 #include <functional>
 #include <optional>
+#include <string>
 #include <thread>
 #include <unordered_map>
 #include <utility>
@@ -31,8 +34,18 @@
 namespace ripple {
 
 template <typename Key>
-std::size_t
-partitioner(Key const& key, std::size_t const numPartitions);
+static std::size_t
+extract(Key const& key)
+{
+ return key;
+}
+
+template <>
+inline std::size_t
+extract(std::string const& key)
+{
+ return ::beast::uhash<>{}(key);
+}
 
 template <
  typename Key,
@@ -211,7 +224,7 @@ class partitioned_unordered_map
  std::size_t
  partitioner(Key const& key) const
  {
- return ripple::partitioner(key, partitions_);
+ return extract(key) % partitions_;
  }
 
  template <class T>

include/xrpl/protocol/AmountConversions.h

@@ -21,8 +21,8 @@
 #define RIPPLE_PROTOCOL_AMOUNTCONVERSION_H_INCLUDED
 
 #include <xrpl/basics/IOUAmount.h>
-#include <xrpl/basics/XRPAmount.h>
 #include <xrpl/protocol/STAmount.h>
+#include <xrpl/protocol/XRPAmount.h>
 
 #include <type_traits>