[v24.2.x] Add rate-rejected connection metric

src/v/net/include/net/server_probe.h

@@ -38,7 +38,13 @@ class server_probe {
 
  void connection_close_error() { ++_connection_close_error; }
 
- void connection_rejected() { ++_connections_rejected; }
+ void connection_rejected_open_limit() {
+ ++_connections_rejected_open_limit;
+ }
+
+ void connection_rejected_rate_limit() {
+ ++_connections_rejected_rate_limit;
+ }
 
  void add_bytes_sent(size_t sent) { _out_bytes += sent; }
 
@@ -56,8 +62,6 @@ class server_probe {
 
  void waiting_for_available_memory() { ++_requests_blocked_memory; }
 
- void timeout_waiting_rate_limit() { ++_declined_new_connections; }
-
  void waiting_for_conection_rate() { ++_connections_wait_rate; }
 
  // metric used to signal a produce request with a timestamp too far into the
@@ -85,11 +89,14 @@ class server_probe {
  uint64_t _service_errors = 0;
  uint32_t _connections = 0;
  uint32_t _connection_close_error = 0;
- uint64_t _connections_rejected = 0;
+ // connections rejected as we hit our "open connections" limit
+ uint64_t _connections_rejected_open_limit = 0;
+ // connections rejected as we hit our connection rate limit and
+ // delaying the connection was not sufficient to stay under the limit
+ uint32_t _connections_rejected_rate_limit = 0;
  uint32_t _corrupted_headers = 0;
  uint32_t _method_not_found_errors = 0;
  uint32_t _requests_blocked_memory = 0;
- uint32_t _declined_new_connections = 0;
  uint32_t _connections_wait_rate = 0;
  uint32_t _produce_bad_create_time = 0;
  friend std::ostream& operator<<(std::ostream& o, const server_probe& p);

src/v/net/probes.cc

@@ -58,9 +58,17 @@ void server_probe::setup_metrics(
  "{}: Number of errors when shutting down the connection", proto))),
  sm::make_counter(
  "connections_rejected",
- [this] { return _connections_rejected; },
+ [this] { return _connections_rejected_open_limit; },
  sm::description(ssx::sformat(
- "{}: Number of connections rejected for hitting connection limits",
+ "{}: Number of connection attempts rejected for hitting open "
+ "connection count limits",
+ proto))),
+ sm::make_counter(
+ "connections_rejected_rate_limit",
+ [this] { return _connections_rejected_rate_limit; },
+ sm::description(ssx::sformat(
+ "{}: Number of connection attempts rejected for hitting "
+ "connection rate limits",
  proto))),
  sm::make_counter(
  "requests_completed",
@@ -166,7 +174,10 @@ std::ostream& operator<<(std::ostream& o, const server_probe& p) {
  << "connects: " << p._connects << ", "
  << "current connections: " << p._connections << ", "
  << "connection close errors: " << p._connection_close_error << ", "
- << "connections rejected: " << p._connections_rejected << ", "
+ << "connections rejected (open limit): "
+ << p._connections_rejected_open_limit << ", "
+ << "connections rejected (rate limit): "
+ << p._connections_rejected_rate_limit << ", "
  << "requests received: " << p._requests_received << ", "
  << "requests completed: " << p._requests_completed << ", "
  << "service errors: " << p._service_errors << ", "
@@ -175,7 +186,6 @@ std::ostream& operator<<(std::ostream& o, const server_probe& p) {
  << "corrupted headers: " << p._corrupted_headers << ", "
  << "method not found errors: " << p._method_not_found_errors << ", "
  << "requests blocked by memory: " << p._requests_blocked_memory << ", "
- << "declined new connections: " << p._declined_new_connections << ", "
  << "connections wait rate: " << p._connections_wait_rate << ", "
  << "produce bad create time: " << p._produce_bad_create_time << ", "
  << "}";

src/v/net/server.cc

@@ -218,11 +218,11 @@ ss::future<ss::stop_iteration> server::accept_finish(
  ar.remote_address.addr());
  if (!cq_units.live()) {
  // Connection limit hit, drop this connection.
- _probe->connection_rejected();
+ _probe->connection_rejected_open_limit();
  vlog(
  _log.warn,
- "Connection limit reached, rejecting {}",
- ar.remote_address.addr());
+ "Open connection limit reached, rejecting {}",
+ ar.remote_address);
  co_return ss::stop_iteration::no;
  }
  }
@@ -248,10 +248,10 @@ ss::future<ss::stop_iteration> server::accept_finish(
  } catch (const std::exception& e) {
  vlog(
  _log.trace,
- "Timeout while waiting free token for connection rate. "
- "addr:{}",
+ "Connection rate limit reached and no token available after "
+ "wait, rejecting {}",
  ar.remote_address);
- _probe->timeout_waiting_rate_limit();
+ _probe->connection_rejected_rate_limit();
  co_return ss::stop_iteration::no;
  }
  }