dgram: initial support for DATAGRAM extension

With this change, we introduce support for the DATAGRAM frame extension
specified in draft-ietf-quic-datagram-01 and draft-schinazi-quic-h3-datagram-04.

Several API methods have been added to the quiche transport library:

* `enable_dgram()` enables the optional QUIC extension for a connection and
  controls internal queue sizes (see below).
* `dgram_max_writable_len()` returns the size of the largest DATAGRAM frame
  payload that can be sent on the connection.
* `dgram_send()` buffers the provided payload data in quiche, a subsequent
  call to `send()` constructs a DATAGRAM frame and QUIC packet.
* `dgram_purge_outgoing()` purges buffered payloads matching a predicate.
* `dgram_recv()` extracts payload data buffered in quiche from a prior call
  to `recv()`.

Internally, a send and receive queue are added to quiche for the purposes of
buffering the payload of DATAGRAM frames. By their specification, these frames
are not flow controlled by QUIC but because of quiche's API design we benefit
some ability to enqueue/dequeue application data independent from the
packetization and network loop. The size of queues is configurable in order
for applications to tailor things to their needs.

Several API methods have been added or updated to the quiche HTTP/3 library,
which internally builds on the transport library:

* `dgram_max_writable_len()` returns the size of the largest DATAGRAM frame
  payload that can be sent on the connection.
* `dgram_send()` buffers the payload for a flow ID.
* `poll()` processes received DATAGRAMs and generates a
  `quiche::h3::Event::Datagram` if there is something to read.
* `dgram_recv()` extracts payload data buffered in quiche.

Previously, quiche-client and quiche-server supported HTTP/0.9, HTTP/3 or both
for any given QUIC connection. During TLS handshake, ALPN would select one of
these and then the applications would create HttpConn objects to handle
request/response. Http09Conn simply invokes quiche transport APIs directly.
Http3Conn invokes quiche transport and H3 APIs.

With DATAGRAM there are many more permutations. Rather than boil the ocean, this
commit introduces support for two modes, with the following CLI parameters that
control how datagrams are used:

```
  --dgram-proto PROTO         DATAGRAM application protocol to use.
  --dgram-count COUNT         Number of DATAGRAMs to send.
  --dgram-data DATA           Data to send for certain types of DATAGRAM application protocol.
```

Valid values for `dgram-proto` are `siduck` and `oneway`.

`siduck` works according to the I-D: the endpoints advertise the ALPN siduck,
siduck-00 and if negotiated, send dgram_count number of DATAGRAM frames. The
client by default sends quack, which can be overridden by the dgram-data
parameter. The server simply attempts to respond to every received DATAGRAM with
${value-ack}; it ignores dgram-count and dgram-data. The client counts sent
quacks and received quack acks and reports this. No HTTP requests or responses
are made in this mode.

`oneway` layers on top of HTTP/3. Endpoints unilaterally send dgram-count amount
of HTTP/3 DATAGRAMS to each other and no accounting is done. Clients send on
flow ID 0, server on flow ID 1. By default the client sends quack, the server
sends brrr. Meanwhile, the apps send requests/responses in accordance with all
existing CLI parameters.

Fixes #430, #573.

Co-authored-by: Marco Mastropaolo <[email protected]> (@xanathar)

examples/http3-client.c

@@ -255,6 +255,9 @@ static void recv_cb(EV_P_ ev_io *w, int revents) {
  fprintf(stderr, "failed to close connection\n");
  }
  break;
+
+ case QUICHE_H3_EVENT_DATAGRAM:
+ break;
  }
 
  quiche_h3_event_free(ev);

examples/http3-client.rs

@@ -258,6 +258,8 @@ fn main() {
  conn.close(true, 0x00, b"kthxbye").unwrap();
  },
 
+ Ok((_flow_id, quiche::h3::Event::Datagram)) => (),
+
  Err(quiche::h3::Error::Done) => {
  break;
  },

examples/http3-server.c

@@ -424,6 +424,9 @@ static void recv_cb(EV_P_ ev_io *w, int revents) {
 
  case QUICHE_H3_EVENT_FINISHED:
  break;
+
+ case QUICHE_H3_EVENT_DATAGRAM:
+ break;
  }
 
  quiche_h3_event_free(ev);

examples/http3-server.rs

@@ -357,6 +357,8 @@ fn main() {
 
  Ok((_stream_id, quiche::h3::Event::Finished)) => (),
 
+ Ok((_flow_id, quiche::h3::Event::Datagram)) => (),
+
  Err(quiche::h3::Error::Done) => {
  break;
  },

extras/nginx/nginx-1.16.patch

@@ -1573,7 +1573,7 @@ new file mode 100644
 index 000000000..358a36528
 --- /dev/null
 +++ b/src/http/v3/ngx_http_v3.c
-@@ -0,0 +1,2228 @@
+@@ -0,0 +1,2231 @@
 +
 +/*
 + * Copyright (C) Cloudflare, Inc.
@@ -2033,6 +2033,9 @@ index 000000000..358a36528
 +
 + break;
 + }
++
++ case QUICHE_H3_EVENT_DATAGRAM:
++ break;
 + }
 +
 + quiche_h3_event_free(ev);

include/quiche.h

@@ -183,6 +183,11 @@ void quiche_config_set_cc_algorithm(quiche_config *config, enum quiche_cc_algori
 // Configures whether to use HyStart++.
 void quiche_config_enable_hystart(quiche_config *config, bool v);
 
+// Configures whether to enable receiving DATAGRAM frames.
+void quiche_config_enable_dgram(quiche_config *config, bool enabled,
+ size_t recv_queue_len,
+ size_t send_queue_len);
+
 // Frees the config object.
 void quiche_config_free(quiche_config *config);
 
@@ -349,6 +354,21 @@ typedef struct {
 // Collects and returns statistics about the connection.
 void quiche_conn_stats(quiche_conn *conn, quiche_stats *out);
 
+// Returns the maximum DATAGRAM payload that can be sent.
+ssize_t quiche_conn_dgram_max_writable_len(quiche_conn *conn);
+
+// Reads the first received DATAGRAM.
+ssize_t quiche_conn_dgram_recv(quiche_conn *conn, uint8_t *buf,
+ size_t buf_len);
+
+// Sends data in a DATAGRAM frame.
+ssize_t quiche_conn_dgram_send(quiche_conn *conn, const uint8_t *buf,
+ size_t buf_len);
+
+// Purges queued outgoing DATAGRAMs matching the predicate.
+void quiche_conn_dgram_purge_outgoing(quiche_conn *conn,
+ bool (*f)(uint8_t *, size_t));
+
 // Frees the connection object.
 void quiche_conn_free(quiche_conn *conn);
 
@@ -438,6 +458,7 @@ enum quiche_h3_event_type {
  QUICHE_H3_EVENT_HEADERS,
  QUICHE_H3_EVENT_DATA,
  QUICHE_H3_EVENT_FINISHED,
+ QUICHE_H3_EVENT_DATAGRAM,
 };
 
 typedef struct Http3Event quiche_h3_event;
@@ -500,6 +521,14 @@ ssize_t quiche_h3_send_body(quiche_h3_conn *conn, quiche_conn *quic_conn,
 ssize_t quiche_h3_recv_body(quiche_h3_conn *conn, quiche_conn *quic_conn,
  uint64_t stream_id, uint8_t *out, size_t out_len);
 
+// Writes data to the DATAGRAM send queue.
+ssize_t quiche_h3_send_dgram(quiche_h3_conn *conn, quiche_conn *quic_conn,
+ uint64_t flow_id, uint8_t *data, size_t data_len);
+
+// Reads data from the DATAGRAM receive queue.
+ssize_t quiche_h3_recv_dgram(quiche_h3_conn *conn, quiche_conn *quic_conn,
+ uint64_t *flow_id, uint8_t *out, size_t out_len);
+
 // Frees the HTTP/3 connection object.
 void quiche_h3_conn_free(quiche_h3_conn *conn);
 

src/dgram.rs

@@ -0,0 +1,105 @@
+// Copyright (C) 2020, Cloudflare, Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+use crate::Error;
+use crate::Result;
+
+use std::collections::VecDeque;
+
+/// Keeps track of DATAGRAM frames.
+#[derive(Default)]
+pub struct DatagramQueue {
+ queue: VecDeque<Vec<u8>>,
+ queue_max_len: usize,
+ queue_bytes_size: usize,
+}
+
+impl DatagramQueue {
+ pub fn new(queue_max_len: usize) -> Self {
+ DatagramQueue {
+ queue: VecDeque::with_capacity(queue_max_len),
+ queue_bytes_size: 0,
+ queue_max_len,
+ }
+ }
+
+ pub fn push(&mut self, data: &[u8]) -> Result<()> {
+ if self.is_full() {
+ return Err(Error::Done);
+ }
+
+ self.queue.push_back(data.to_vec());
+ self.queue_bytes_size += data.len();
+ Ok(())
+ }
+
+ pub fn peek_front_len(&self) -> Option<usize> {
+ self.queue.front().map(|d| d.len())
+ }
+
+ pub fn peek_front_bytes(&self, buf: &mut [u8], len: usize) -> Result<usize> {
+ match self.queue.front() {
+ Some(d) => {
+ let len = std::cmp::min(len, d.len());
+ if buf.len() < len {
+ return Err(Error::BufferTooShort);
+ }
+
+ buf[..len].copy_from_slice(&d[..len]);
+ Ok(len)
+ },
+
+ None => Err(Error::Done),
+ }
+ }
+
+ pub fn pop(&mut self) -> Option<Vec<u8>> {
+ if let Some(d) = self.queue.pop_front() {
+ self.queue_bytes_size = self.queue_bytes_size.saturating_sub(d.len());
+ return Some(d);
+ }
+
+ None
+ }
+
+ pub fn has_pending(&self) -> bool {
+ !self.queue.is_empty()
+ }
+
+ pub fn purge<F: Fn(&[u8]) -> bool>(&mut self, f: F) {
+ self.queue.retain(|d| !f(d));
+ self.queue_bytes_size =
+ self.queue.iter().fold(0, |total, d| total + d.len());
+ }
+
+ pub fn is_full(&self) -> bool {
+ self.queue.len() == self.queue_max_len
+ }
+
+ pub fn byte_size(&self) -> usize {
+ self.queue_bytes_size
+ }
+}

src/ffi.rs

@@ -241,6 +241,14 @@ pub extern fn quiche_config_enable_hystart(config: &mut Config, v: bool) {
  config.enable_hystart(v);
 }
 
+#[no_mangle]
+pub extern fn quiche_config_enable_dgram(
+ config: &mut Config, enabled: bool, recv_queue_len: size_t,
+ send_queue_len: size_t,
+) {
+ config.enable_dgram(enabled, recv_queue_len, send_queue_len);
+}
+
 #[no_mangle]
 pub extern fn quiche_config_free(config: *mut Config) {
  unsafe { Box::from_raw(config) };
@@ -733,6 +741,63 @@ pub extern fn quiche_conn_stats(conn: &Connection, out: &mut Stats) {
  out.delivery_rate = stats.delivery_rate;
 }
 
+#[no_mangle]
+pub extern fn quiche_conn_dgram_max_writable_len(conn: &Connection) -> ssize_t {
+ match conn.dgram_max_writable_len() {
+ None => Error::Done.to_c(),
+
+ Some(v) => v as ssize_t,
+ }
+}
+
+#[no_mangle]
+pub extern fn quiche_conn_dgram_send(
+ conn: &mut Connection, buf: *const u8, buf_len: size_t,
+) -> ssize_t {
+ if buf_len > <ssize_t>::max_value() as usize {
+ panic!("The provided buffer is too large");
+ }
+
+ let buf = unsafe { slice::from_raw_parts(buf, buf_len) };
+
+ match conn.dgram_send(buf) {
+ Ok(_) => buf_len as ssize_t,
+
+ Err(e) => e.to_c(),
+ }
+}
+
+#[no_mangle]
+pub extern fn quiche_conn_dgram_recv(
+ conn: &mut Connection, out: *mut u8, out_len: size_t,
+) -> ssize_t {
+ if out_len > <ssize_t>::max_value() as usize {
+ panic!("The provided buffer is too large");
+ }
+
+ let out = unsafe { slice::from_raw_parts_mut(out, out_len) };
+
+ let out_len = match conn.dgram_recv(out) {
+ Ok(v) => v,
+
+ Err(e) => return e.to_c(),
+ };
+
+ out_len as ssize_t
+}
+
+#[no_mangle]
+pub extern fn quiche_conn_dgram_purge_outgoing(
+ conn: &mut Connection, f: extern fn(*const u8, size_t) -> bool,
+) {
+ conn.dgram_purge_outgoing(|d: &[u8]| -> bool {
+ let ptr: *const u8 = d.as_ptr();
+ let len: size_t = d.len();
+
+ f(ptr, len)
+ });
+}
+
 #[no_mangle]
 pub extern fn quiche_conn_free(conn: *mut Connection) {
  unsafe { Box::from_raw(conn) };