Virtualization - May 2023 - [PATCH net-next 4/5] virtio_ring: Introduce DMA pre-handler

"private" of buffer page is currently used for big mode to chain
pages.
But in mergeable mode, that offset of page could mean something else,
e.g. when page_pool page is used instead. So excluding mergeable mode to
avoid such a problem.

Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
---
 drivers/net/virtio_net.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 5a7f7a76b920..c5dca0d92e64 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -497,7 +497,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 			return NULL;
 
 		page = (struct page *)page->private;
-		if (page)
+		if (!vi->mergeable_rx_bufs && page)
 			give_pages(rq, page);
 		goto ok;
 	}
-- 
2.31.1

The implementation at the moment uses one page per packet in both the
normal and XDP path. In addition, introducing a module parameter to enable
or disable the usage of page pool (disabled by default).

In single-core vm testing environments, it gives a modest performance gain
in the normal path.
  Upstream codebase: 47.5 Gbits/sec
  Upstream codebase + page_pool support: 50.2 Gbits/sec

In multi-core vm testing environments, The most significant performance
gain is observed in XDP cpumap:
  Upstream codebase: 1.38 Gbits/sec
  Upstream codebase + page_pool support: 9.74 Gbits/sec

With this foundation, we can further integrate page pool fragmentation and
DMA map/unmap support.

Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
---
 drivers/net/virtio_net.c | 188 ++++++++++++++++++++++++++++++---------
 1 file changed, 146 insertions(+), 42 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index c5dca0d92e64..99c0ca0c1781 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -31,6 +31,9 @@ module_param(csum, bool, 0444);
 module_param(gso, bool, 0444);
 module_param(napi_tx, bool, 0644);
 
+static bool page_pool_enabled;
+module_param(page_pool_enabled, bool, 0400);
+
 /* FIXME: MTU in config. */
 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
 #define GOOD_COPY_LEN	128
@@ -159,6 +162,9 @@ struct receive_queue {
 	/* Chain pages by the private ptr. */
 	struct page *pages;
 
+	/* Page pool */
+	struct page_pool *page_pool;
+
 	/* Average packet length for mergeable receive buffers. */
 	struct ewma_pkt_len mrg_avg_pkt_len;
 
@@ -459,6 +465,14 @@ static struct sk_buff *virtnet_build_skb(void *buf,
unsigned int buflen,
 	return skb;
 }
 
+static void virtnet_put_page(struct receive_queue *rq, struct page *page)
+{
+	if (rq->page_pool)
+		page_pool_put_full_page(rq->page_pool, page, true);
+	else
+		put_page(page);
+}
+
 /* Called from bottom half context */
 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 				   struct receive_queue *rq,
@@ -555,7 +569,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 	hdr = skb_vnet_hdr(skb);
 	memcpy(hdr, hdr_p, hdr_len);
 	if (page_to_free)
-		put_page(page_to_free);
+		virtnet_put_page(rq, page_to_free);
 
 	return skb;
 }
@@ -802,7 +816,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
 	return ret;
 }
 
-static void put_xdp_frags(struct xdp_buff *xdp)
+static void put_xdp_frags(struct xdp_buff *xdp, struct receive_queue *rq)
 {
 	struct skb_shared_info *shinfo;
 	struct page *xdp_page;
@@ -812,7 +826,7 @@ static void put_xdp_frags(struct xdp_buff *xdp)
 		shinfo = xdp_get_shared_info_from_buff(xdp);
 		for (i = 0; i < shinfo->nr_frags; i++) {
 			xdp_page = skb_frag_page(&shinfo->frags[i]);
-			put_page(xdp_page);
+			virtnet_put_page(rq, xdp_page);
 		}
 	}
 }
@@ -903,7 +917,11 @@ static struct page *xdp_linearize_page(struct receive_queue
*rq,
 	if (page_off + *len + tailroom > PAGE_SIZE)
 		return NULL;
 
-	page = alloc_page(GFP_ATOMIC);
+	if (rq->page_pool)
+		page = page_pool_dev_alloc_pages(rq->page_pool);
+	else
+		page = alloc_page(GFP_ATOMIC);
+
 	if (!page)
 		return NULL;
 
@@ -926,21 +944,24 @@ static struct page *xdp_linearize_page(struct
receive_queue *rq,
 		 * is sending packet larger than the MTU.
 		 */
 		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
-			put_page(p);
+			virtnet_put_page(rq, p);
 			goto err_buf;
 		}
 
 		memcpy(page_address(page) + page_off,
 		       page_address(p) + off, buflen);
 		page_off += buflen;
-		put_page(p);
+		virtnet_put_page(rq, p);
 	}
 
 	/* Headroom does not contribute to packet length */
 	*len = page_off - VIRTIO_XDP_HEADROOM;
 	return page;
 err_buf:
-	__free_pages(page, 0);
+	if (rq->page_pool)
+		page_pool_put_full_page(rq->page_pool, page, true);
+	else
+		__free_pages(page, 0);
 	return NULL;
 }
 
@@ -1144,7 +1165,7 @@ static void mergeable_buf_free(struct receive_queue *rq,
int num_buf,
 		}
 		stats->bytes += len;
 		page = virt_to_head_page(buf);
-		put_page(page);
+		virtnet_put_page(rq, page);
 	}
 }
 
@@ -1264,7 +1285,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device
*dev,
 		cur_frag_size = truesize;
 		xdp_frags_truesz += cur_frag_size;
 		if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) {
-			put_page(page);
+			virtnet_put_page(rq, page);
 			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 				 dev->name, len, (unsigned long)(truesize - room));
 			dev->stats.rx_length_errors++;
@@ -1283,7 +1304,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device
*dev,
 	return 0;
 
 err:
-	put_xdp_frags(xdp);
+	put_xdp_frags(xdp, rq);
 	return -EINVAL;
 }
 
@@ -1344,7 +1365,10 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 		if (*len + xdp_room > PAGE_SIZE)
 			return NULL;
 
-		xdp_page = alloc_page(GFP_ATOMIC);
+		if (rq->page_pool)
+			xdp_page = page_pool_dev_alloc_pages(rq->page_pool);
+		else
+			xdp_page = alloc_page(GFP_ATOMIC);
 		if (!xdp_page)
 			return NULL;
 
@@ -1354,7 +1378,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 
 	*frame_sz = PAGE_SIZE;
 
-	put_page(*page);
+	virtnet_put_page(rq, *page);
 
 	*page = xdp_page;
 
@@ -1400,6 +1424,8 @@ static struct sk_buff *receive_mergeable_xdp(struct
net_device *dev,
 		head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
 		if (unlikely(!head_skb))
 			break;
+		if (rq->page_pool)
+			skb_mark_for_recycle(head_skb);
 		return head_skb;
 
 	case XDP_TX:
@@ -1410,10 +1436,10 @@ static struct sk_buff *receive_mergeable_xdp(struct
net_device *dev,
 		break;
 	}
 
-	put_xdp_frags(&xdp);
+	put_xdp_frags(&xdp, rq);
 
 err_xdp:
-	put_page(page);
+	virtnet_put_page(rq, page);
 	mergeable_buf_free(rq, num_buf, dev, stats);
 
 	stats->xdp_drops++;
@@ -1467,6 +1493,9 @@ static struct sk_buff *receive_mergeable(struct net_device
*dev,
 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
 	curr_skb = head_skb;
 
+	if (rq->page_pool)
+		skb_mark_for_recycle(curr_skb);
+
 	if (unlikely(!curr_skb))
 		goto err_skb;
 	while (--num_buf) {
@@ -1509,6 +1538,8 @@ static struct sk_buff *receive_mergeable(struct net_device
*dev,
 			curr_skb = nskb;
 			head_skb->truesize += nskb->truesize;
 			num_skb_frags = 0;
+			if (rq->page_pool)
+				skb_mark_for_recycle(curr_skb);
 		}
 		if (curr_skb != head_skb) {
 			head_skb->data_len += len;
@@ -1517,7 +1548,7 @@ static struct sk_buff *receive_mergeable(struct net_device
*dev,
 		}
 		offset = buf - page_address(page);
 		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
-			put_page(page);
+			virtnet_put_page(rq, page);
 			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
 					     len, truesize);
 		} else {
@@ -1530,7 +1561,7 @@ static struct sk_buff *receive_mergeable(struct net_device
*dev,
 	return head_skb;
 
 err_skb:
-	put_page(page);
+	virtnet_put_page(rq, page);
 	mergeable_buf_free(rq, num_buf, dev, stats);
 
 err_buf:
@@ -1737,31 +1768,40 @@ static int add_recvbuf_mergeable(struct virtnet_info
*vi,
 	 * disabled GSO for XDP, it won't be a big issue.
 	 */
 	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
-	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
-		return -ENOMEM;
+	if (rq->page_pool) {
+		struct page *page;
 
-	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
-	buf += headroom; /* advance address leaving hole at front of pkt */
-	get_page(alloc_frag->page);
-	alloc_frag->offset += len + room;
-	hole = alloc_frag->size - alloc_frag->offset;
-	if (hole < len + room) {
-		/* To avoid internal fragmentation, if there is very likely not
-		 * enough space for another buffer, add the remaining space to
-		 * the current buffer.
-		 * XDP core assumes that frame_size of xdp_buff and the length
-		 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
-		 */
-		if (!headroom)
-			len += hole;
-		alloc_frag->offset += hole;
-	}
+		page = page_pool_dev_alloc_pages(rq->page_pool);
+		if (unlikely(!page))
+			return -ENOMEM;
+		buf = (char *)page_address(page);
+		buf += headroom; /* advance address leaving hole at front of pkt */
+	} else {
+		if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
+			return -ENOMEM;
 
+		buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+		buf += headroom; /* advance address leaving hole at front of pkt */
+		get_page(alloc_frag->page);
+		alloc_frag->offset += len + room;
+		hole = alloc_frag->size - alloc_frag->offset;
+		if (hole < len + room) {
+			/* To avoid internal fragmentation, if there is very likely not
+			 * enough space for another buffer, add the remaining space to
+			 * the current buffer.
+			 * XDP core assumes that frame_size of xdp_buff and the length
+			 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
+			 */
+			if (!headroom)
+				len += hole;
+			alloc_frag->offset += hole;
+		}
+	}
 	sg_init_one(rq->sg, buf, len);
 	ctx = mergeable_len_to_ctx(len + room, headroom);
 	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 	if (err < 0)
-		put_page(virt_to_head_page(buf));
+		virtnet_put_page(rq, virt_to_head_page(buf));
 
 	return err;
 }
@@ -1994,8 +2034,15 @@ static int virtnet_enable_queue_pair(struct virtnet_info
*vi, int qp_index)
 	if (err < 0)
 		return err;
 
-	err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
-					 MEM_TYPE_PAGE_SHARED, NULL);
+	if (vi->rq[qp_index].page_pool)
+		err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
+						 MEM_TYPE_PAGE_POOL,
+						 vi->rq[qp_index].page_pool);
+	else
+		err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
+						 MEM_TYPE_PAGE_SHARED,
+						 NULL);
+
 	if (err < 0)
 		goto err_xdp_reg_mem_model;
 
@@ -2951,6 +2998,7 @@ static void virtnet_get_strings(struct net_device *dev,
u32 stringset, u8 *data)
 				ethtool_sprintf(&p, "tx_queue_%u_%s", i,
 						virtnet_sq_stats_desc[j].desc);
 		}
+		page_pool_ethtool_stats_get_strings(p);
 		break;
 	}
 }
@@ -2962,12 +3010,30 @@ static int virtnet_get_sset_count(struct net_device
*dev, int sset)
 	switch (sset) {
 	case ETH_SS_STATS:
 		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
-					       VIRTNET_SQ_STATS_LEN);
+					       VIRTNET_SQ_STATS_LEN +
+						(page_pool_enabled && vi->mergeable_rx_bufs ?
+						 page_pool_ethtool_stats_get_count() : 0));
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
+static void virtnet_get_page_pool_stats(struct net_device *dev, u64 *data)
+{
+#ifdef CONFIG_PAGE_POOL_STATS
+	struct virtnet_info *vi = netdev_priv(dev);
+	struct page_pool_stats pp_stats = {};
+	int i;
+
+	for (i = 0; i < vi->curr_queue_pairs; i++) {
+		if (!vi->rq[i].page_pool)
+			continue;
+		page_pool_get_stats(vi->rq[i].page_pool, &pp_stats);
+	}
+	page_pool_ethtool_stats_get(data, &pp_stats);
+#endif /* CONFIG_PAGE_POOL_STATS */
+}
+
 static void virtnet_get_ethtool_stats(struct net_device *dev,
 				      struct ethtool_stats *stats, u64 *data)
 {
@@ -3003,6 +3069,8 @@ static void virtnet_get_ethtool_stats(struct net_device
*dev,
 		} while (u64_stats_fetch_retry(&sq->stats.syncp, start));
 		idx += VIRTNET_SQ_STATS_LEN;
 	}
+
+	virtnet_get_page_pool_stats(dev, &data[idx]);
 }
 
 static void virtnet_get_channels(struct net_device *dev,
@@ -3623,6 +3691,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
 	for (i = 0; i < vi->max_queue_pairs; i++) {
 		__netif_napi_del(&vi->rq[i].napi);
 		__netif_napi_del(&vi->sq[i].napi);
+		if (vi->rq[i].page_pool)
+			page_pool_destroy(vi->rq[i].page_pool);
 	}
 
 	/* We called __netif_napi_del(),
@@ -3679,12 +3749,19 @@ static void virtnet_rq_free_unused_buf(struct virtqueue
*vq, void *buf)
 	struct virtnet_info *vi = vq->vdev->priv;
 	int i = vq2rxq(vq);
 
-	if (vi->mergeable_rx_bufs)
-		put_page(virt_to_head_page(buf));
-	else if (vi->big_packets)
+	if (vi->mergeable_rx_bufs) {
+		if (vi->rq[i].page_pool) {
+			page_pool_put_full_page(vi->rq[i].page_pool,
+						virt_to_head_page(buf),
+						true);
+		} else {
+			put_page(virt_to_head_page(buf));
+		}
+	} else if (vi->big_packets) {
 		give_pages(&vi->rq[i], buf);
-	else
+	} else {
 		put_page(virt_to_head_page(buf));
+	}
 }
 
 static void free_unused_bufs(struct virtnet_info *vi)
@@ -3718,6 +3795,26 @@ static void virtnet_del_vqs(struct virtnet_info *vi)
 	virtnet_free_queues(vi);
 }
 
+static void virtnet_alloc_page_pool(struct receive_queue *rq)
+{
+	struct virtio_device *vdev = rq->vq->vdev;
+
+	struct page_pool_params pp_params = {
+		.order = 0,
+		.pool_size = rq->vq->num_max,
+		.nid = dev_to_node(vdev->dev.parent),
+		.dev = vdev->dev.parent,
+		.offset = 0,
+	};
+
+	rq->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rq->page_pool)) {
+		dev_warn(&vdev->dev, "page pool creation failed: %ld\n",
+			 PTR_ERR(rq->page_pool));
+		rq->page_pool = NULL;
+	}
+}
+
 /* How large should a single buffer be so a queue full of these can fit at
  * least one full packet?
  * Logic below assumes the mergeable buffer header is used.
@@ -3801,6 +3898,13 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
 		vi->rq[i].vq = vqs[rxq2vq(i)];
 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
 		vi->sq[i].vq = vqs[txq2vq(i)];
+
+		if (page_pool_enabled && vi->mergeable_rx_bufs)
+			virtnet_alloc_page_pool(&vi->rq[i]);
+		else
+			dev_warn(&vi->vdev->dev,
+				 "page pool only support mergeable mode\n");
+
 	}
 
 	/* run here: ret == 0. */
-- 
2.31.1

To further enhance performance, implement page pool fragmentation
support and introduce a module parameter to enable or disable it.

In single-core vm testing environments, there is an additional performance
gain observed in the normal path compared to the one packet per page
approach.
  Upstream codebase: 47.5 Gbits/sec
  Upstream codebase with page pool: 50.2 Gbits/sec
  Upstream codebase with page pool fragmentation support: 52.3 Gbits/sec

There is also some performance gain for XDP cpumap.
  Upstream codebase: 1.38 Gbits/sec
  Upstream codebase with page pool: 9.74 Gbits/sec
  Upstream codebase with page pool fragmentation: 10.3 Gbits/sec

Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
---
 drivers/net/virtio_net.c | 72 ++++++++++++++++++++++++++++++----------
 1 file changed, 55 insertions(+), 17 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 99c0ca0c1781..ac40b8c66c59 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -32,7 +32,9 @@ module_param(gso, bool, 0444);
 module_param(napi_tx, bool, 0644);
 
 static bool page_pool_enabled;
+static bool page_pool_frag;
 module_param(page_pool_enabled, bool, 0400);
+module_param(page_pool_frag, bool, 0400);
 
 /* FIXME: MTU in config. */
 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
@@ -909,23 +911,32 @@ static struct page *xdp_linearize_page(struct
receive_queue *rq,
 				       struct page *p,
 				       int offset,
 				       int page_off,
-				       unsigned int *len)
+				       unsigned int *len,
+					   unsigned int *pp_frag_offset)
 {
 	int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 	struct page *page;
+	unsigned int pp_frag_offset_val;
 
 	if (page_off + *len + tailroom > PAGE_SIZE)
 		return NULL;
 
 	if (rq->page_pool)
-		page = page_pool_dev_alloc_pages(rq->page_pool);
+		if (rq->page_pool->p.flags & PP_FLAG_PAGE_FRAG)
+			page = page_pool_dev_alloc_frag(rq->page_pool, pp_frag_offset,
+							PAGE_SIZE);
+		else
+			page = page_pool_dev_alloc_pages(rq->page_pool);
 	else
 		page = alloc_page(GFP_ATOMIC);
 
 	if (!page)
 		return NULL;
 
-	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
+	pp_frag_offset_val = pp_frag_offset ? *pp_frag_offset : 0;
+
+	memcpy(page_address(page) + page_off + pp_frag_offset_val,
+	       page_address(p) + offset, *len);
 	page_off += *len;
 
 	while (--*num_buf) {
@@ -948,7 +959,7 @@ static struct page *xdp_linearize_page(struct receive_queue
*rq,
 			goto err_buf;
 		}
 
-		memcpy(page_address(page) + page_off,
+		memcpy(page_address(page) + page_off + pp_frag_offset_val,
 		       page_address(p) + off, buflen);
 		page_off += buflen;
 		virtnet_put_page(rq, p);
@@ -1029,7 +1040,7 @@ static struct sk_buff *receive_small_xdp(struct net_device
*dev,
 			SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 		xdp_page = xdp_linearize_page(rq, &num_buf, page,
 					      offset, header_offset,
-					      &tlen);
+					      &tlen, NULL);
 		if (!xdp_page)
 			goto err_xdp;
 
@@ -1323,6 +1334,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 	struct page *xdp_page;
 	unsigned int xdp_room;
+	unsigned int page_frag_offset = 0;
 
 	/* Transient failure which in theory could occur if
 	 * in-flight packets from before XDP was enabled reach
@@ -1356,7 +1368,8 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 		xdp_page = xdp_linearize_page(rq, num_buf,
 					      *page, offset,
 					      VIRTIO_XDP_HEADROOM,
-					      len);
+					      len,
+						  &page_frag_offset);
 		if (!xdp_page)
 			return NULL;
 	} else {
@@ -1366,14 +1379,19 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 			return NULL;
 
 		if (rq->page_pool)
-			xdp_page = page_pool_dev_alloc_pages(rq->page_pool);
+			if (rq->page_pool->p.flags & PP_FLAG_PAGE_FRAG)
+				xdp_page = page_pool_dev_alloc_frag(rq->page_pool,
+								    &page_frag_offset, PAGE_SIZE);
+			else
+				xdp_page = page_pool_dev_alloc_pages(rq->page_pool);
 		else
 			xdp_page = alloc_page(GFP_ATOMIC);
+
 		if (!xdp_page)
 			return NULL;
 
-		memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM,
-		       page_address(*page) + offset, *len);
+		memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM +
+				page_frag_offset, page_address(*page) + offset, *len);
 	}
 
 	*frame_sz = PAGE_SIZE;
@@ -1382,7 +1400,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info
*vi,
 
 	*page = xdp_page;
 
-	return page_address(*page) + VIRTIO_XDP_HEADROOM;
+	return page_address(*page) + VIRTIO_XDP_HEADROOM + page_frag_offset;
 }
 
 static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
@@ -1762,6 +1780,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
 	void *ctx;
 	int err;
 	unsigned int len, hole;
+	unsigned int pp_frag_offset;
 
 	/* Extra tailroom is needed to satisfy XDP's assumption. This
 	 * means rx frags coalescing won't work, but consider we've
@@ -1769,13 +1788,29 @@ static int add_recvbuf_mergeable(struct virtnet_info
*vi,
 	 */
 	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
 	if (rq->page_pool) {
-		struct page *page;
+		if (rq->page_pool->p.flags & PP_FLAG_PAGE_FRAG) {
+			if (unlikely(!page_pool_dev_alloc_frag(rq->page_pool,
+							       &pp_frag_offset, len + room)))
+				return -ENOMEM;
+			buf = (char *)page_address(rq->page_pool->frag_page) +
+				pp_frag_offset;
+			buf += headroom; /* advance address leaving hole at front of pkt */
+			hole = (PAGE_SIZE << rq->page_pool->p.order)
+				- rq->page_pool->frag_offset;
+			if (hole < len + room) {
+				if (!headroom)
+					len += hole;
+				rq->page_pool->frag_offset += hole;
+			}
+		} else {
+			struct page *page;
 
-		page = page_pool_dev_alloc_pages(rq->page_pool);
-		if (unlikely(!page))
-			return -ENOMEM;
-		buf = (char *)page_address(page);
-		buf += headroom; /* advance address leaving hole at front of pkt */
+			page = page_pool_dev_alloc_pages(rq->page_pool);
+			if (unlikely(!page))
+				return -ENOMEM;
+			buf = (char *)page_address(page);
+			buf += headroom; /* advance address leaving hole at front of pkt */
+		}
 	} else {
 		if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
 			return -ENOMEM;
@@ -3800,13 +3835,16 @@ static void virtnet_alloc_page_pool(struct receive_queue
*rq)
 	struct virtio_device *vdev = rq->vq->vdev;
 
 	struct page_pool_params pp_params = {
-		.order = 0,
+		.order = page_pool_frag ? SKB_FRAG_PAGE_ORDER : 0,
 		.pool_size = rq->vq->num_max,
 		.nid = dev_to_node(vdev->dev.parent),
 		.dev = vdev->dev.parent,
 		.offset = 0,
 	};
 
+	if (page_pool_frag)
+		pp_params.flags |= PP_FLAG_PAGE_FRAG;
+
 	rq->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(rq->page_pool)) {
 		dev_warn(&vdev->dev, "page pool creation failed: %ld\n",
-- 
2.31.1

Currently, DMA operations of virtio devices' data buffer are encapsulated
within the underlying virtqueue implementation. DMA map/unmap operations
are performed for each data buffer attached to/detached from the virtqueue,
which is transparent and invisible to the higher-level virtio device
drivers. This encapsulation makes it not viable for device drivers to
introduce certain mechanisms, such as page pool, that require explicit
management of DMA map/unmap. Therefore, by inserting a pre-handler before
the generic DMA map/unmap operations, virtio device drivers have the
opportunity to participate in DMA operations.

Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
---
 drivers/virtio/virtio_ring.c | 73 +++++++++++++++++++++++++++++++++---
 include/linux/virtio.h       | 18 +++++++++
 2 files changed, 85 insertions(+), 6 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c5310eaf8b46..a99641260555 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -213,6 +213,9 @@ struct vring_virtqueue {
 	bool last_add_time_valid;
 	ktime_t last_add_time;
 #endif
+
+	/* DMA mapping Pre-handler for virtio device driver */
+	struct virtqueue_pre_dma_ops *pre_dma_ops;
 };
 
 static struct virtqueue *__vring_new_virtqueue(unsigned int index,
@@ -369,6 +372,19 @@ static dma_addr_t vring_map_one_sg(const struct
vring_virtqueue *vq,
 		return (dma_addr_t)sg_phys(sg);
 	}
 
+	/* Allow virtio drivers to perform customized mapping operation, and
+	 * fallback to the generic path if it fails to handle the mapping.
+	 */
+	if (vq->pre_dma_ops && vq->pre_dma_ops->map_page) {
+		dma_addr_t addr;
+
+		addr = vq->pre_dma_ops->map_page(vring_dma_dev(vq),
+				sg_page(sg), sg->offset, sg->length,
+				direction, 0);
+		if (addr)
+			return addr;
+	}
+
 	/*
 	 * We can't use dma_map_sg, because we don't use scatterlists in
 	 * the way it expects (we don't guarantee that the scatterlist
@@ -432,6 +448,15 @@ static void vring_unmap_one_split_indirect(const struct
vring_virtqueue *vq,
 
 	flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
 
+	if (vq->pre_dma_ops && vq->pre_dma_ops->unmap_page) {
+		if (vq->pre_dma_ops->unmap_page(vring_dma_dev(vq),
+					virtio64_to_cpu(vq->vq.vdev, desc->addr),
+					virtio32_to_cpu(vq->vq.vdev, desc->len),
+					(flags & VRING_DESC_F_WRITE) ?
+					DMA_FROM_DEVICE : DMA_TO_DEVICE, 0))
+			return;
+	}
+
 	dma_unmap_page(vring_dma_dev(vq),
 		       virtio64_to_cpu(vq->vq.vdev, desc->addr),
 		       virtio32_to_cpu(vq->vq.vdev, desc->len),
@@ -456,14 +481,22 @@ static unsigned int vring_unmap_one_split(const struct
vring_virtqueue *vq,
 				 extra[i].len,
 				 (flags & VRING_DESC_F_WRITE) ?
 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	} else {
-		dma_unmap_page(vring_dma_dev(vq),
-			       extra[i].addr,
-			       extra[i].len,
-			       (flags & VRING_DESC_F_WRITE) ?
-			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
+		goto out;
+	} else if (vq->pre_dma_ops && vq->pre_dma_ops->unmap_page) {
+		if (vq->pre_dma_ops->unmap_page(vring_dma_dev(vq),
+					extra[i].addr,
+					extra[i].len,
+					(flags & VRING_DESC_F_WRITE) ?
+					DMA_FROM_DEVICE : DMA_TO_DEVICE, 0))
+			goto out;
 	}
 
+	dma_unmap_page(vring_dma_dev(vq),
+			extra[i].addr,
+			extra[i].len,
+			(flags & VRING_DESC_F_WRITE) ?
+			DMA_FROM_DEVICE : DMA_TO_DEVICE);
+
 out:
 	return extra[i].next;
 }
@@ -1206,10 +1239,19 @@ static void vring_unmap_extra_packed(const struct
vring_virtqueue *vq,
 				 (flags & VRING_DESC_F_WRITE) ?
 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
 	} else {
+		if (vq->pre_dma_ops && vq->pre_dma_ops->unmap_page) {
+			if (vq->pre_dma_ops->unmap_page(vring_dma_dev(vq),
+						extra->addr,
+						extra->len,
+						(flags & VRING_DESC_F_WRITE) ?
+						DMA_FROM_DEVICE : DMA_TO_DEVICE, 0))
+				return;
+		}
 		dma_unmap_page(vring_dma_dev(vq),
 			       extra->addr, extra->len,
 			       (flags & VRING_DESC_F_WRITE) ?
 			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
+
 	}
 }
 
@@ -1223,6 +1265,15 @@ static void vring_unmap_desc_packed(const struct
vring_virtqueue *vq,
 
 	flags = le16_to_cpu(desc->flags);
 
+	if (vq->pre_dma_ops && vq->pre_dma_ops->unmap_page) {
+		if (vq->pre_dma_ops->unmap_page(vring_dma_dev(vq),
+					le64_to_cpu(desc->addr),
+					le32_to_cpu(desc->len),
+					(flags & VRING_DESC_F_WRITE) ?
+					DMA_FROM_DEVICE : DMA_TO_DEVICE, 0))
+			return;
+	}
+
 	dma_unmap_page(vring_dma_dev(vq),
 		       le64_to_cpu(desc->addr),
 		       le32_to_cpu(desc->len),
@@ -2052,6 +2103,7 @@ static struct virtqueue *vring_create_virtqueue_packed(
 	vq->packed_ring = true;
 	vq->dma_dev = dma_dev;
 	vq->use_dma_api = vring_use_dma_api(vdev);
+	vq->pre_dma_ops = NULL;
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)
&&
 		!context;
@@ -2541,6 +2593,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned
int index,
 #endif
 	vq->dma_dev = dma_dev;
 	vq->use_dma_api = vring_use_dma_api(vdev);
+	vq->pre_dma_ops = NULL;
 
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)
&&
 		!context;
@@ -2945,4 +2998,12 @@ const struct vring *virtqueue_get_vring(const struct
virtqueue *vq)
 }
 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
 
+/* The virtio device driver can register its own DMA map/unmap pre-handler. */
+void virtqueue_register_pre_dma_ops(struct virtqueue *vq,
+		struct virtqueue_pre_dma_ops *pre_dma_ops)
+{
+	to_vvq(vq)->pre_dma_ops = pre_dma_ops;
+}
+EXPORT_SYMBOL_GPL(virtqueue_register_pre_dma_ops);
+
 MODULE_LICENSE("GPL");
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index b93238db94e3..1d5755b5e03f 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -9,6 +9,7 @@
 #include <linux/device.h>
 #include <linux/mod_devicetable.h>
 #include <linux/gfp.h>
+#include <linux/dma-map-ops.h>
 
 /**
  * struct virtqueue - a queue to register buffers for sending or receiving.
@@ -203,4 +204,21 @@ void unregister_virtio_driver(struct virtio_driver *drv);
 #define module_virtio_driver(__virtio_driver) \
 	module_driver(__virtio_driver, register_virtio_driver, \
 			unregister_virtio_driver)
+/**
+ * struct virtqueue_pre_dma_ops - DMA pre-handler for virtio device driver
+ * @map_page: map a single page of memory for DMA
+ * @unmap_page: unmap a single page of memory for DMA
+ */
+struct virtqueue_pre_dma_ops {
+	dma_addr_t (*map_page)(struct device *dev, struct page *page,
+			unsigned long offset, size_t size,
+			enum dma_data_direction dir, unsigned long attrs);
+	bool (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
+			size_t size, enum dma_data_direction dir,
+			unsigned long attrs);
+};
+
+void virtqueue_register_pre_dma_ops(struct virtqueue *vq,
+		struct virtqueue_pre_dma_ops *pre_dma_ops);
+
 #endif /* _LINUX_VIRTIO_H */
-- 
2.31.1

Adding a DMA pre-handler that utilizes page pool for managing DMA mappings.
When IOMMU is enabled, turning on the page_pool_dma_map module parameter to
select page pool for DMA mapping management gives a significant reduction
in the overhead caused by DMA mappings.

In testing environments with a single core vm and qemu emulated IOMMU,
significant performance improvements can be observed:
  Upstream codebase: 1.76 Gbits/sec
  Upstream codebase with page pool fragmentation support: 1.81 Gbits/sec
  Upstream codebase with page pool fragmentation and DMA support: 19.3
  Gbits/sec

Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
---
 drivers/net/virtio_net.c | 55 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index ac40b8c66c59..73cc4f9fe4fa 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -22,6 +22,7 @@
 #include <net/route.h>
 #include <net/xdp.h>
 #include <net/net_failover.h>
+#include <linux/iommu.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -33,8 +34,10 @@ module_param(napi_tx, bool, 0644);
 
 static bool page_pool_enabled;
 static bool page_pool_frag;
+static bool page_pool_dma_map;
 module_param(page_pool_enabled, bool, 0400);
 module_param(page_pool_frag, bool, 0400);
+module_param(page_pool_dma_map, bool, 0400);
 
 /* FIXME: MTU in config. */
 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
@@ -3830,6 +3833,49 @@ static void virtnet_del_vqs(struct virtnet_info *vi)
 	virtnet_free_queues(vi);
 }
 
+static dma_addr_t virtnet_pp_dma_map_page(struct device *dev, struct page
*page,
+					  unsigned long offset, size_t size,
+					  enum dma_data_direction dir, unsigned long attrs)
+{
+	struct page *head_page;
+
+	if (dir != DMA_FROM_DEVICE)
+		return 0;
+
+	head_page = compound_head(page);
+	return page_pool_get_dma_addr(head_page)
+		+ (page - head_page) * PAGE_SIZE
+		+ offset;
+}
+
+static bool virtnet_pp_dma_unmap_page(struct device *dev, dma_addr_t
dma_handle,
+				      size_t size, enum dma_data_direction dir,
+				      unsigned long attrs)
+{
+	phys_addr_t phys;
+
+	/* Handle only the RX direction, and sync the DMA memory only if it's not
+	 * a DMA coherent architecture.
+	 */
+	if (dir != DMA_FROM_DEVICE)
+		return false;
+
+	if (dev_is_dma_coherent(dev))
+		return true;
+
+	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
+	if (WARN_ON(!phys))
+		return false;
+
+	arch_sync_dma_for_cpu(phys, size, dir);
+	return true;
+}
+
+static struct virtqueue_pre_dma_ops virtnet_pp_pre_dma_ops = {
+	.map_page = virtnet_pp_dma_map_page,
+	.unmap_page = virtnet_pp_dma_unmap_page,
+};
+
 static void virtnet_alloc_page_pool(struct receive_queue *rq)
 {
 	struct virtio_device *vdev = rq->vq->vdev;
@@ -3845,6 +3891,15 @@ static void virtnet_alloc_page_pool(struct receive_queue
*rq)
 	if (page_pool_frag)
 		pp_params.flags |= PP_FLAG_PAGE_FRAG;
 
+	/* Consider using page pool DMA support only when DMA API is used. */
+	if (virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM) &&
+	    page_pool_dma_map) {
+		pp_params.flags |= PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+		pp_params.dma_dir = DMA_FROM_DEVICE;
+		pp_params.max_len = PAGE_SIZE << pp_params.order;
+		virtqueue_register_pre_dma_ops(rq->vq, &virtnet_pp_pre_dma_ops);
+	}
+
 	rq->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(rq->page_pool)) {
 		dev_warn(&vdev->dev, "page pool creation failed: %ld\n",
-- 
2.31.1

On Fri, May 26, 2023 at 1:46?PM Liang Chen <liangchen.linux at gmail.com>
wrote:
>
> "private" of buffer page is currently used for big mode to chain
pages.
> But in mergeable mode, that offset of page could mean something else,
> e.g. when page_pool page is used instead. So excluding mergeable mode to
> avoid such a problem.
If this issue happens only in the case of page_pool, it would be
better to squash it there.

Thanks
>
> Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
> ---
>  drivers/net/virtio_net.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 5a7f7a76b920..c5dca0d92e64 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -497,7 +497,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info
*vi,
>                         return NULL;
>
>                 page = (struct page *)page->private;
> -               if (page)
> +               if (!vi->mergeable_rx_bufs && page)
>                         give_pages(rq, page);
>                 goto ok;
>         }
> --
> 2.31.1
>

On Fri, May 26, 2023 at 01:46:17PM +0800, Liang Chen
wrote:
> "private" of buffer page is currently used for big mode to chain
pages.
> But in mergeable mode, that offset of page could mean something else,
> e.g. when page_pool page is used instead. So excluding mergeable mode to
> avoid such a problem.
> 
> Signed-off-by: Liang Chen <liangchen.linux at gmail.com>
Ugh the subject makes it looks like current code has a problem
but I don't think so because I don't think anything besides
big packets uses page->private.

The reason patch is needed is because follow up patches
use page_pool.
pls adjust commit log and subject to make all this clear.

> ---
>  drivers/net/virtio_net.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 5a7f7a76b920..c5dca0d92e64 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -497,7 +497,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info
*vi,
>  			return NULL;
>  
>  		page = (struct page *)page->private;
> -		if (page)
> +		if (!vi->mergeable_rx_bufs && page)
To be safe let's limit to big packets too:

	if (!vi->mergeable_rx_bufs && vi->big_packets && page)


>  			give_pages(rq, page);
>  		goto ok;
>  	}
> -- 
> 2.31.1