feat: support Concise Transaction Identifier (CTID)

(XLS-37) (XRPLF#4418)

The XLS-37 CTID (Concise Transaction ID) is a network-aware tx
identifier which provides a way to efficiently locate a specific
transaction without relying on transaction hashes. A CTID encodes the
sequence number of the ledger that includes the tx, the transaction's
index in that ledger, and the network ID. With the CTID, users can
identify transactions and confirm their results. This applies even for
transactions on sidechains, which may be difficult to find with only a
transaction hash.

Additionally, CTIDs require less storage space than transaction hashes,
which can be beneficial for databases storing millions of transactions.

The XLS-37 specification can be found at:
  XRPLF/XRPL-Standards#111

Add support for running a node on a different network. There is a new
error code, `rpcWRONG_NETWORK`, returned when the requested CTID's
network ID does not match the node's network ID. The error message looks
like:

  Wrong network. You should submit this request to a node running on
  NetworkID: <net_id>

* Add RPC support for the CTID format
  * tx - you can specify "ctid", which is the CTID (16 hex digits, in a
    string, always starting with "C")
  * When retrieving a tx, the "ctid" may be returned
* Add support for encoding, decoding, and validating CTIDs
* Add tests

---------

Co-authored-by: Rome Reginelli <[email protected]>
Co-authored-by: Denis Angell <[email protected]>

---------

Co-authored-by: Rome Reginelli <[email protected]>
Co-authored-by: Elliot Lee <[email protected]>
Co-authored-by: Denis Angell <[email protected]>
Signed-off-by: Manoj Doshi <[email protected]>

Builds/CMake/RippledCore.cmake

@@ -768,6 +768,7 @@ if (tests)
  src/test/app/HashRouter_test.cpp
  src/test/app/LedgerHistory_test.cpp
  src/test/app/LedgerLoad_test.cpp
+ src/test/app/LedgerMaster_test.cpp
  src/test/app/LedgerReplay_test.cpp
  src/test/app/LoadFeeTrack_test.cpp
  src/test/app/Manifest_test.cpp

src/ripple/app/ledger/LedgerMaster.h

@@ -292,6 +292,10 @@ class LedgerMaster : public AbstractFetchPackContainer
  std::optional<LedgerIndex>
  minSqlSeq();
 
+ // Iff a txn exists at the specified ledger and offset then return its txnid
+ std::optional<uint256>
+ txnIdFromIndex(uint32_t ledgerSeq, uint32_t txnIndex);
+
 private:
  void
  setValidLedger(std::shared_ptr<Ledger const> const& l);

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -2369,4 +2369,25 @@ LedgerMaster::minSqlSeq()
  return app_.getRelationalDatabase().getMinLedgerSeq();
 }
 
+std::optional<uint256>
+LedgerMaster::txnIdFromIndex(uint32_t ledgerSeq, uint32_t txnIndex)
+{
+ uint32_t first = 0, last = 0;
+
+ if (!getValidatedRange(first, last) || last < ledgerSeq)
+ return {};
+
+ auto const lgr = getLedgerBySeq(ledgerSeq);
+ if (!lgr || lgr->txs.empty())
+ return {};
+
+ for (auto it = lgr->txs.begin(); it != lgr->txs.end(); ++it)
+ if (it->first && it->second &&
+ it->second->isFieldPresent(sfTransactionIndex) &&
+ it->second->getFieldU32(sfTransactionIndex) == txnIndex)
+ return it->first->getTransactionID();
+
+ return {};
+}
+
 } // namespace ripple

src/ripple/net/impl/RPCCall.cpp

@@ -1096,7 +1096,11 @@ class RPCParser
  jvRequest[jss::max_ledger] = jvParams[2u + offset].asString();
  }
 
- jvRequest[jss::transaction] = jvParams[0u].asString();
+ if (jvParams[0u].asString().length() == 16)
+ jvRequest[jss::ctid] = jvParams[0u].asString();
+ else
+ jvRequest[jss::transaction] = jvParams[0u].asString();
+
  return jvRequest;
  }
 

src/ripple/protocol/ErrorCodes.h

@@ -47,8 +47,8 @@ enum error_code_i {
  rpcJSON_RPC = 2,
  rpcFORBIDDEN = 3,
 
+ rpcWRONG_NETWORK = 4,
  // Misc failure
- // unused 4,
  // unused 5,
  rpcNO_PERMISSION = 6,
  rpcNO_EVENTS = 7,

src/ripple/protocol/jss.h

@@ -226,6 +226,7 @@ JSS(converge_time_s); // out: NetworkOPs
 JSS(cookie); // out: NetworkOPs
 JSS(count); // in: AccountTx*, ValidatorList
 JSS(counters); // in/out: retrieve counters
+JSS(ctid); // in/out: Tx RPC
 JSS(currency_a); // out: BookChanges
 JSS(currency_b); // out: BookChanges
 JSS(currentShard); // out: NodeToShardStatus

src/ripple/rpc/CTID.h

@@ -0,0 +1,88 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of rippled: https:/ripple/rippled
+ Copyright (c) 2019 Ripple Labs Inc.
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_RPC_CTID_H_INCLUDED
+#define RIPPLE_RPC_CTID_H_INCLUDED
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/regex.hpp>
+#include <optional>
+#include <regex>
+#include <sstream>
+
+namespace ripple {
+
+namespace RPC {
+
+inline std::optional<std::string>
+encodeCTID(
+ uint32_t ledger_seq,
+ uint16_t txn_index,
+ uint16_t network_id) noexcept
+{
+ if (ledger_seq > 0x0FFF'FFFF)
+ return {};
+
+ uint64_t ctidValue =
+ ((0xC000'0000ULL + static_cast<uint64_t>(ledger_seq)) << 32) +
+ (static_cast<uint64_t>(txn_index) << 16) + network_id;
+
+ std::stringstream buffer;
+ buffer << std::hex << std::uppercase << std::setfill('0') << std::setw(16)
+ << ctidValue;
+ return {buffer.str()};
+}
+
+template <typename T>
+inline std::optional<std::tuple<uint32_t, uint16_t, uint16_t>>
+decodeCTID(const T ctid) noexcept
+{
+ uint64_t ctidValue{0};
+ if constexpr (
+ std::is_same_v<T, std::string> || std::is_same_v<T, char*> ||
+ std::is_same_v<T, const char*> || std::is_same_v<T, std::string_view>)
+ {
+ std::string const ctidString(ctid);
+
+ if (ctidString.length() != 16)
+ return {};
+
+ if (!boost::regex_match(ctidString, boost::regex("^[0-9A-F]+$")))
+ return {};
+
+ ctidValue = std::stoull(ctidString, nullptr, 16);
+ }
+ else if constexpr (std::is_integral_v<T>)
+ ctidValue = ctid;
+ else
+ return {};
+
+ if ((ctidValue & 0xF000'0000'0000'0000ULL) != 0xC000'0000'0000'0000ULL)
+ return {};
+
+ uint32_t ledger_seq = (ctidValue >> 32) & 0xFFFF'FFFUL;
+ uint16_t txn_index = (ctidValue >> 16) & 0xFFFFU;
+ uint16_t network_id = ctidValue & 0xFFFFU;
+ return {{ledger_seq, txn_index, network_id}};
+}
+
+} // namespace RPC
+} // namespace ripple
+
+#endif

src/ripple/rpc/handlers/Tx.cpp

@@ -25,18 +25,17 @@
 #include <ripple/net/RPCErr.h>
 #include <ripple/protocol/ErrorCodes.h>
 #include <ripple/protocol/jss.h>
+#include <ripple/rpc/CTID.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/DeliveredAmount.h>
 #include <ripple/rpc/GRPCHandlers.h>
 #include <ripple/rpc/NFTSyntheticSerializer.h>
 #include <ripple/rpc/impl/RPCHelpers.h>
+#include <charconv>
+#include <regex>
 
 namespace ripple {
 
-// {
-// transaction: <hex>
-// }
-
 static bool
 isValidated(LedgerMaster& ledgerMaster, std::uint32_t seq, uint256 const& hash)
 {
@@ -54,12 +53,14 @@ struct TxResult
  Transaction::pointer txn;
  std::variant<std::shared_ptr<TxMeta>, Blob> meta;
  bool validated = false;
+ std::optional<std::string> ctid;
  TxSearched searchedAll;
 };
 
 struct TxArgs
 {
- uint256 hash;
+ std::optional<uint256> hash;
+ std::optional<std::pair<uint32_t, uint16_t>> ctid;
  bool binary = false;
  std::optional<std::pair<uint32_t, uint32_t>> ledgerRange;
 };
@@ -73,11 +74,19 @@ doTxPostgres(RPC::Context& context, TxArgs const& args)
  Throw<std::runtime_error>(
  "Called doTxPostgres yet not in reporting mode");
  }
+
  TxResult res;
  res.searchedAll = TxSearched::unknown;
 
+ if (!args.hash)
+ return {
+ res,
+ {rpcNOT_IMPL,
+ "Use of CTIDs on reporting mode is not currently supported."}};
+
  JLOG(context.j.debug()) << "Fetching from postgres";
- Transaction::Locator locator = Transaction::locate(args.hash, context.app);
+ Transaction::Locator locator =
+ Transaction::locate(*(args.hash), context.app);
 
  std::pair<std::shared_ptr<STTx const>, std::shared_ptr<STObject const>>
  pair;
@@ -127,7 +136,7 @@ doTxPostgres(RPC::Context& context, TxArgs const& args)
  else
  {
  res.meta = std::make_shared<TxMeta>(
- args.hash, res.txn->getLedger(), *meta);
+ *(args.hash), res.txn->getLedger(), *meta);
  }
  res.validated = true;
  return {res, rpcSUCCESS};
@@ -168,7 +177,7 @@ doTxPostgres(RPC::Context& context, TxArgs const& args)
 }
 
 std::pair<TxResult, RPC::Status>
-doTxHelp(RPC::Context& context, TxArgs const& args)
+doTxHelp(RPC::Context& context, TxArgs args)
 {
  if (context.app.config().reporting())
  return doTxPostgres(context, args);
@@ -196,15 +205,28 @@ doTxHelp(RPC::Context& context, TxArgs const& args)
  std::pair<std::shared_ptr<Transaction>, std::shared_ptr<TxMeta>>;
 
  result.searchedAll = TxSearched::unknown;
-
  std::variant<TxPair, TxSearched> v;
+
+ if (args.ctid)
+ {
+ args.hash = context.app.getLedgerMaster().txnIdFromIndex(
+ args.ctid->first, args.ctid->second);
+
+ if (args.hash)
+ range =
+ ClosedInterval<uint32_t>(args.ctid->first, args.ctid->second);
+ }
+
+ if (!args.hash)
+ return {result, rpcTXN_NOT_FOUND};
+
  if (args.ledgerRange)
  {
- v = context.app.getMasterTransaction().fetch(args.hash, range, ec);
+ v = context.app.getMasterTransaction().fetch(*(args.hash), range, ec);
  }
  else
  {
- v = context.app.getMasterTransaction().fetch(args.hash, ec);
+ v = context.app.getMasterTransaction().fetch(*(args.hash), ec);
  }
 
  if (auto e = std::get_if<TxSearched>(&v))
@@ -246,6 +268,15 @@ doTxHelp(RPC::Context& context, TxArgs const& args)
  }
  result.validated = isValidated(
  context.ledgerMaster, ledger->info().seq, ledger->info().hash);
+
+ // compute outgoing CTID
+ uint32_t lgrSeq = ledger->info().seq;
+ uint32_t txnIdx = meta->getAsObject().getFieldU32(sfTransactionIndex);
+ uint32_t netID = context.app.config().NETWORK_ID;
+
+ if (txnIdx <= 0xFFFFU && netID < 0xFFFFU && lgrSeq < 0x0FFF'FFFFUL)
+ result.ctid =
+ RPC::encodeCTID(lgrSeq, (uint16_t)txnIdx, (uint16_t)netID);
  }
 
  return {result, rpcSUCCESS};
@@ -301,6 +332,9 @@ populateJsonResponse(
  }
  }
  response[jss::validated] = result.validated;
+
+ if (result.ctid)
+ response[jss::ctid] = *(result.ctid);
  }
  return response;
 }
@@ -313,13 +347,39 @@ doTxJson(RPC::JsonContext& context)
 
  // Deserialize and validate JSON arguments
 
- if (!context.params.isMember(jss::transaction))
+ TxArgs args;
+
+ if (context.params.isMember(jss::transaction) &&
+ context.params.isMember(jss::ctid))
+ // specifying both is ambiguous
  return rpcError(rpcINVALID_PARAMS);
 
- TxArgs args;
+ if (context.params.isMember(jss::transaction))
+ {
+ uint256 hash;
+ if (!hash.parseHex(context.params[jss::transaction].asString()))
+ return rpcError(rpcNOT_IMPL);
+ args.hash = hash;
+ }
+ else if (context.params.isMember(jss::ctid))
+ {
+ auto ctid = RPC::decodeCTID(context.params[jss::ctid].asString());
+ if (!ctid)
+ return rpcError(rpcINVALID_PARAMS);
 
- if (!args.hash.parseHex(context.params[jss::transaction].asString()))
- return rpcError(rpcNOT_IMPL);
+ auto const [lgr_seq, txn_idx, net_id] = *ctid;
+ if (net_id != context.app.config().NETWORK_ID)
+ {
+ std::stringstream out;
+ out << "Wrong network. You should submit this request to a node "
+ "running on NetworkID: "
+ << net_id;
+ return RPC::make_error(rpcWRONG_NETWORK, out.str());
+ }
+ args.ctid = {lgr_seq, txn_idx};
+ }
+ else
+ return rpcError(rpcINVALID_PARAMS);
 
  args.binary = context.params.isMember(jss::binary) &&
  context.params[jss::binary].asBool();

src/ripple/rpc/impl/RPCHelpers.cpp

@@ -34,6 +34,7 @@
 #include <ripple/rpc/DeliveredAmount.h>
 #include <ripple/rpc/impl/RPCHelpers.h>
 #include <boost/algorithm/string/case_conv.hpp>
+#include <regex>
 
 namespace ripple {
 namespace RPC {