The first draft of complete basic XDP series for IDPF upstream

drivers/net/ethernet/intel/idpf/idpf.h

@@ -16,6 +16,9 @@ struct idpf_vport_max_q;
 #include <linux/bitfield.h>
 #include <linux/sctp.h>
 #include <linux/ethtool.h>
+#include <linux/bpf_trace.h>
+#include <linux/filter.h>
+#include <linux/bpf.h>
 #include <net/gro.h>
 #include <linux/dim.h>
 
@@ -376,6 +379,13 @@ struct idpf_vport {
  struct idpf_queue **txqs;
  bool crc_enable;
 
+ int num_xdp_txq;
+ int num_xdp_rxq;
+ int num_xdp_complq;
+ int xdp_txq_offset;
+ int xdp_rxq_offset;
+ int xdp_complq_offset;
+
  u16 num_rxq;
  u16 num_bufq;
  u32 rxq_desc_count;
@@ -423,6 +433,7 @@ struct idpf_vport {
  * @__IDPF_USER_FLAGS_NBITS: Must be last
  */
 enum idpf_user_flags {
+ __IDPF_PRIV_FLAGS_HDR_SPLIT = 0,
  __IDPF_PROMISC_UC = 32,
  __IDPF_PROMISC_MC,
 
@@ -466,6 +477,8 @@ struct idpf_vport_user_config_data {
  u16 num_req_rx_qs;
  u32 num_req_txq_desc;
  u32 num_req_rxq_desc;
+ /* Duplicated in queue structure for performance reasons */
+ struct bpf_prog *xdp_prog;
  DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS);
  struct list_head mac_filter_list;
 };
@@ -684,6 +697,18 @@ static inline int idpf_is_queue_model_split(u16 q_model)
  return q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT;
 }
 
+/**
+ * idpf_xdp_is_prog_ena - check if there is an XDP program on adapter
+ * @vport: vport to check
+ */
+static inline bool idpf_xdp_is_prog_ena(struct idpf_vport *vport)
+{
+ if (!vport->adapter)
+ return false;
+
+ return !!vport->adapter->vport_config[vport->idx]->user_config.xdp_prog;
+}
+
 #define idpf_is_cap_ena(adapter, field, flag) \
  idpf_is_capability_ena(adapter, false, field, flag)
 #define idpf_is_cap_ena_all(adapter, field, flag) \
@@ -939,6 +964,7 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter, u32 op,
 int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op,
  u16 msg_size, u8 *msg);
 void idpf_set_ethtool_ops(struct net_device *netdev);
+void idpf_vport_set_hsplit(struct idpf_vport *vport, bool ena);
 int idpf_vport_alloc_max_qs(struct idpf_adapter *adapter,
  struct idpf_vport_max_q *max_q);
 void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter,

drivers/net/ethernet/intel/idpf/idpf_ethtool.c

@@ -900,6 +900,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev,
 
  if (!txq)
  idpf_add_empty_queue_stats(&data, qtype);
+ else if (test_bit(__IDPF_Q_XDP, txq->flags))
+ continue;
  else
  idpf_add_queue_stats(&data, txq);
  }

drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h

@@ -184,13 +184,17 @@ struct idpf_base_tx_desc {
  __le64 qw1; /* type_cmd_offset_bsz_l2tag1 */
 }; /* read used with buffer queues */
 
-struct idpf_splitq_tx_compl_desc {
+struct idpf_splitq_4b_tx_compl_desc {
  /* qid=[10:0] comptype=[13:11] rsvd=[14] gen=[15] */
  __le16 qid_comptype_gen;
  union {
  __le16 q_head; /* Queue head */
  __le16 compl_tag; /* Completion tag */
  } q_head_compl_tag;
+}; /* writeback used with completion queues */
+
+struct idpf_splitq_tx_compl_desc {
+ struct idpf_splitq_4b_tx_compl_desc common;
  u8 ts[3];
  u8 rsvd; /* Reserved */
 }; /* writeback used with completion queues */

drivers/net/ethernet/intel/idpf/idpf_lib.c

@@ -1057,6 +1057,27 @@ static void idpf_vport_dealloc(struct idpf_vport *vport)
  adapter->next_vport = idpf_get_free_slot(adapter);
 }
 
+/**
+ * idpf_vport_set_hsplit - enable or disable header split on a given vport
+ * @vport: virtual port
+ * @ena: flag controlling header split, On (true) or Off (false)
+ */
+void idpf_vport_set_hsplit(struct idpf_vport *vport, bool ena)
+{
+ struct idpf_vport_user_config_data *config_data;
+
+ config_data = &vport->adapter->vport_config[vport->idx]->user_config;
+ if (!ena) {
+ clear_bit(__IDPF_PRIV_FLAGS_HDR_SPLIT, config_data->user_flags);
+ return;
+ }
+
+ if (idpf_is_cap_ena_all(vport->adapter, IDPF_HSPLIT_CAPS,
+ IDPF_CAP_HSPLIT) &&
+ idpf_is_queue_model_split(vport->rxq_model))
+ set_bit(__IDPF_PRIV_FLAGS_HDR_SPLIT, config_data->user_flags);
+}
+
 /**
  * idpf_vport_alloc - Allocates the next available struct vport in the adapter
  * @adapter: board private structure
@@ -1234,13 +1255,18 @@ static void idpf_restore_features(struct idpf_vport *vport)
  */
 static int idpf_set_real_num_queues(struct idpf_vport *vport)
 {
- int err;
+ int num_txq, err;
 
  err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq);
  if (err)
  return err;
 
- return netif_set_real_num_tx_queues(vport->netdev, vport->num_txq);
+ if (idpf_xdp_is_prog_ena(vport))
+ num_txq = vport->num_txq - vport->num_xdp_txq;
+ else
+ num_txq = vport->num_txq;
+
+ return netif_set_real_num_tx_queues(vport->netdev, num_txq);
 }
 
 /**
@@ -1353,6 +1379,15 @@ static int idpf_vport_open(struct idpf_vport *vport, bool alloc_res)
 
  idpf_rx_init_buf_tail(vport);
 
+ if (idpf_xdp_is_prog_ena(vport)) {
+ err = idpf_xdp_rxq_info_init_all(vport);
+ if (err) {
+ dev_err(&adapter->pdev->dev, "Failed to initialize XDP info for vport %u, %d\n",
+ vport->vport_id, err);
+ goto intr_deinit;
+ }
+ }
+
  err = idpf_send_config_queues_msg(vport);
  if (err) {
  dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n",
@@ -2192,10 +2227,18 @@ static int idpf_change_mtu(struct net_device *netdev, int new_mtu)
  idpf_vport_ctrl_lock(netdev);
  vport = idpf_netdev_to_vport(netdev);
 
+ if (idpf_xdp_is_prog_ena(vport) && new_mtu > IDPF_XDP_MAX_MTU) {
+ netdev_err(netdev, "New MTU value is not valid. The maximum MTU value is %d.\n",
+ IDPF_XDP_MAX_MTU);
+ err = -EINVAL;
+ goto unlock_exit;
+ }
+
  netdev->mtu = new_mtu;
 
  err = idpf_initiate_soft_reset(vport, IDPF_SR_MTU_CHANGE);
 
+unlock_exit:
  idpf_vport_ctrl_unlock(netdev);
 
  return err;
@@ -2262,6 +2305,194 @@ static netdev_features_t idpf_features_check(struct sk_buff *skb,
  return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
 }
 
+/**
+ * idpf_copy_xdp_prog_to_qs - set pointers to xdp program for each Rx queue
+ * @vport: vport to setup XDP for
+ * @xdp_prog: XDP program that should be copied to all Rx queues
+ */
+static void
+idpf_copy_xdp_prog_to_qs(struct idpf_vport *vport, struct bpf_prog *xdp_prog)
+{
+ int i;
+
+ for (i = 0; i < vport->num_rxq_grp; i++) {
+ struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
+ struct idpf_queue *q;
+ u16 j, num_rxq;
+
+ if (idpf_is_queue_model_split(vport->rxq_model))
+ num_rxq = rx_qgrp->splitq.num_rxq_sets;
+ else
+ num_rxq = rx_qgrp->singleq.num_rxq;
+
+ for (j = 0; j < num_rxq; j++) {
+ if (idpf_is_queue_model_split(vport->rxq_model))
+ q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
+ else
+ q = rx_qgrp->singleq.rxqs[j];
+ WRITE_ONCE(q->xdp_prog, xdp_prog);
+ }
+
+ if (!idpf_is_queue_model_split(vport->rxq_model))
+ continue;
+
+ for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
+ q = &rx_qgrp->splitq.bufq_sets[j].bufq;
+ WRITE_ONCE(q->xdp_prog, xdp_prog);
+ }
+ }
+}
+
+/**
+ * idpf_xdp_reconfig_queues - reconfigure queues after the XDP setup
+ * @vport: vport to load or unload XDP for
+ */
+static int idpf_xdp_reconfig_queues(struct idpf_vport *vport)
+{
+ int err;
+
+ err = idpf_vport_adjust_qs(vport);
+ if (err) {
+ netdev_err(vport->netdev,
+ "Could not adjust queue number for XDP\n");
+ return err;
+ }
+ idpf_vport_calc_num_q_desc(vport);
+
+ err = idpf_vport_queues_alloc(vport);
+ if (err) {
+ netdev_err(vport->netdev,
+ "Could not allocate queues for XDP\n");
+ return err;
+ }
+
+ err = idpf_send_add_queues_msg(vport, vport->num_txq,
+ vport->num_complq,
+ vport->num_rxq, vport->num_bufq);
+ if (err) {
+ netdev_err(vport->netdev,
+ "Could not add queues for XDP, VC message sent failed\n");
+ return err;
+ }
+
+ idpf_vport_alloc_vec_indexes(vport);
+
+ return 0;
+}
+
+/**
+ * idpf_assign_bpf_prog - Assign a given BPF program to vport
+ * @current_prog: pointer to XDP program in user config data
+ * @prog: BPF program to be assigned to vport
+ */
+static void idpf_assign_bpf_prog(struct bpf_prog **current_prog,
+ struct bpf_prog *prog)
+{
+ struct bpf_prog *old_prog;
+
+ old_prog = xchg(current_prog, prog);
+ if (old_prog)
+ bpf_prog_put(old_prog);
+}
+
+/**
+ * idpf_xdp_setup_prog - Add or remove XDP eBPF program
+ * @vport: vport to setup XDP for
+ * @prog: XDP program
+ * @extack: netlink extended ack
+ */
+static int
+idpf_xdp_setup_prog(struct idpf_vport *vport, struct bpf_prog *prog,
+ struct netlink_ext_ack *extack)
+{
+ struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
+ int frame_size = vport->netdev->mtu;
+ bool needs_reconfig, vport_is_up;
+ struct bpf_prog **current_prog;
+ u16 idx = vport->idx;
+ int err;
+
+ if (frame_size > IDPF_XDP_MAX_MTU ||
+ frame_size > vport->bufq_size[0]) {
+ NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP");
+ return -EOPNOTSUPP;
+ }
+
+ vport_is_up = np->state == __IDPF_VPORT_UP;
+
+ current_prog = &vport->adapter->vport_config[idx]->user_config.xdp_prog;
+ needs_reconfig = !!(*current_prog) != !!prog;
+
+ if (!needs_reconfig) {
+ idpf_copy_xdp_prog_to_qs(vport, prog);
+ idpf_assign_bpf_prog(current_prog, prog);
+
+ return 0;
+ }
+
+ if (!vport_is_up) {
+ idpf_send_delete_queues_msg(vport);
+ } else {
+ set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags);
+ idpf_vport_stop(vport);
+ }
+
+ idpf_deinit_rss(vport);
+
+ if (!*current_prog && prog) {
+ netdev_warn(vport->netdev,
+ "Setting up XDP disables header split\n");
+ idpf_vport_set_hsplit(vport, false);
+ } else {
+ idpf_vport_set_hsplit(vport, true);
+ }
+
+ idpf_assign_bpf_prog(current_prog, prog);
+
+ err = idpf_xdp_reconfig_queues(vport);
+ if (err) {
+ netdev_err(vport->netdev,
+ "Could not reconfigure the queues after XDP setup\n");
+ return err;
+ }
+
+ if (vport_is_up) {
+ err = idpf_vport_open(vport, false);
+ if (err) {
+ netdev_err(vport->netdev,
+ "Could not re-open the vport after XDP setup\n");
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * idpf_xdp - implements XDP handler
+ * @netdev: netdevice
+ * @xdp: XDP command
+ */
+static int idpf_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
+{
+ struct idpf_vport *vport;
+ int err;
+
+ idpf_vport_ctrl_lock(netdev);
+ vport = idpf_netdev_to_vport(netdev);
+
+ switch (xdp->command) {
+ case XDP_SETUP_PROG:
+ err = idpf_xdp_setup_prog(vport, xdp->prog, xdp->extack);
+ break;
+ default:
+ err = -EINVAL;
+ }
+
+ idpf_vport_ctrl_unlock(netdev);
+ return err;
+}
+
 /**
  * idpf_set_mac - NDO callback to set port mac address
  * @netdev: network interface device structure
@@ -2362,6 +2593,7 @@ static const struct net_device_ops idpf_netdev_ops_splitq = {
  .ndo_get_stats64 = idpf_get_stats64,
  .ndo_set_features = idpf_set_features,
  .ndo_tx_timeout = idpf_tx_timeout,
+ .ndo_bpf = idpf_xdp,
 };
 
 static const struct net_device_ops idpf_netdev_ops_singleq = {