search for: add_recvbuf_merg

...ven the latency, refill buffer right after being used. Sign-off-by: Shirley Ma <xma at us.ibm.com> --- diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 0c7321c..c8201d4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -429,6 +429,22 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp) return err; } +static bool fill_one(struct virtio_net *vi, gfp_t gfp) +{ + int err; + + if (vi->mergeable_rx_bufs) + err = add_recvbuf_mergeable(vi, gfp); + else if (vi->big_packets) + err = add_recvbuf_big(vi, gfp); + else + err = add_recvbuf_...

...ven the latency, refill buffer right after being used. Sign-off-by: Shirley Ma <xma at us.ibm.com> --- diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 0c7321c..c8201d4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -429,6 +429,22 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp) return err; } +static bool fill_one(struct virtio_net *vi, gfp_t gfp) +{ + int err; + + if (vi->mergeable_rx_bufs) + err = add_recvbuf_mergeable(vi, gfp); + else if (vi->big_packets) + err = add_recvbuf_big(vi, gfp); + else + err = add_recvbuf_...

....com> --- drivers/net/virtio_net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 98dd75b665a5..3f3aa8308918 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -1184,7 +1184,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, char *buf; void *ctx; int err; - unsigned int len, hole; + unsigned int len, hole, truesize; /* Extra tailroom is needed to satisfy XDP's assumption. This * means rx frags coalescing won't work, but consider we've @@ -1194,6 +1194,7 @@ static i...

...droom + sizeof(struct padded_vnet_hdr); sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); /* chain first in list head */ @@ -823,24 +830,27 @@ static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len) return ALIGN(len, MERGEABLE_BUFFER_ALIGN); } -static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) +static int add_recvbuf_mergeable(struct virtnet_info *vi, + struct receive_queue *rq, gfp_t gfp) { struct page_frag *alloc_frag = &rq->alloc_frag; + unsigned int headroom = vi->headroom; char *buf; unsigned long ctx; int err; unsig...

...droom + sizeof(struct padded_vnet_hdr); sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); /* chain first in list head */ @@ -823,24 +830,27 @@ static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len) return ALIGN(len, MERGEABLE_BUFFER_ALIGN); } -static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) +static int add_recvbuf_mergeable(struct virtnet_info *vi, + struct receive_queue *rq, gfp_t gfp) { struct page_frag *alloc_frag = &rq->alloc_frag; + unsigned int headroom = vi->headroom; char *buf; unsigned long ctx; int err; unsig...

...t.c | 6 ++++-- > 1 file changed, 4 insertions(+), 2 deletions(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 98dd75b665a5..3f3aa8308918 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -1184,7 +1184,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, > char *buf; > void *ctx; > int err; > - unsigned int len, hole; > + unsigned int len, hole, truesize; > > /* Extra tailroom is needed to satisfy XDP's assumption. This > * means rx frags coalescing won't work, but consider...

...t.c | 6 ++++-- > 1 file changed, 4 insertions(+), 2 deletions(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 98dd75b665a5..3f3aa8308918 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -1184,7 +1184,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, > char *buf; > void *ctx; > int err; > - unsigned int len, hole; > + unsigned int len, hole, truesize; > > /* Extra tailroom is needed to satisfy XDP's assumption. This > * means rx frags coalescing won't work, but consider...

...hdr_len); + hdr = ALIGN(VNET_SKB_PAD + sizeof(struct skb_shared_info), + MERGEABLE_BUFFER_ALIGN); + + len = hdr + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), + 500 /* TODO */, PAGE_SIZE - hdr); return ALIGN(len, MERGEABLE_BUFFER_ALIGN); } @@ -626,7 +648,10 @@ static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) return -ENOMEM; - buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; + BUILD_BUG_ON(VNET_SKB_BUG); + + buf = (char *)page_address(alloc_frag->page) + alloc_frag->o...

...hdr_len); + hdr = ALIGN(VNET_SKB_PAD + sizeof(struct skb_shared_info), + MERGEABLE_BUFFER_ALIGN); + + len = hdr + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), + 500 /* TODO */, PAGE_SIZE - hdr); return ALIGN(len, MERGEABLE_BUFFER_ALIGN); } @@ -626,7 +648,10 @@ static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp))) return -ENOMEM; - buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; + BUILD_BUG_ON(VNET_SKB_BUG); + + buf = (char *)page_address(alloc_frag->page) + alloc_frag->o...

We tried to reserve space for vnet header before xdp.data_hard_start. But this is useless since the packet could be modified by XDP which may invalidate the information stored in the header and there's no way for XDP to know the existence of the vnet header currently. So let's just not reserve space for vnet header in this case. Cc: Jesper Dangaard Brouer <brouer at redhat.com>

We tried to reserve space for vnet header before xdp.data_hard_start. But this is useless since the packet could be modified by XDP which may invalidate the information stored in the header and there's no way for XDP to know the existence of the vnet header currently. So let's just not reserve space for vnet header in this case. Cc: Jesper Dangaard Brouer <brouer at redhat.com>

...ceive path after XDP is loaded. + */ + if (unlikely(hdr->hdr.gso_type)) + return NULL; + + /* Now XDP core assumes frag size is PAGE_SIZE, but buffers + * with headroom may add hole in truesize, which + * make their length exceed PAGE_SIZE. So we disabled the + * hole mechanism for xdp. See add_recvbuf_mergeable(). + */ + *frame_sz = truesize; + + /* This happens when headroom is not enough because + * of the buffer was prefilled before XDP is set. + * This should only happen for the first several packets. + * In fact, vq reset can be used here to help us clean up + * the prefilled buffers, but m...

...*/ > + if (unlikely(hdr->hdr.gso_type)) > + return NULL; > + > + /* Now XDP core assumes frag size is PAGE_SIZE, but buffers > + * with headroom may add hole in truesize, which > + * make their length exceed PAGE_SIZE. So we disabled the > + * hole mechanism for xdp. See add_recvbuf_mergeable(). > + */ > + *frame_sz = truesize; > + > + /* This happens when headroom is not enough because > + * of the buffer was prefilled before XDP is set. > + * This should only happen for the first several packets. > + * In fact, vq reset can be used here to help us clean u...

The following series applies to net-next. The following series changes the low memory paths in virtio_net to not disable NAPI while waiting in the allocator in the slow path. It attempts to rectify some performance problems we've seen where the network performance drops significantly when memory is low. The working theory is that the disabling of NAPI while allocations are occuring in the

The following series applies to net-next. The following series changes the low memory paths in virtio_net to not disable NAPI while waiting in the allocator in the slow path. It attempts to rectify some performance problems we've seen where the network performance drops significantly when memory is low. The working theory is that the disabling of NAPI while allocations are occuring in the

...struct receive_queue *rq, } if (vi->mergeable_rx_bufs) - skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len); + skb = receive_mergeable(dev, vi, rq, buf, ctx, len); else if (vi->big_packets) skb = receive_big(dev, vi, rq, buf, len); else @@ -869,7 +858,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, struct page_frag *alloc_frag = &rq->alloc_frag; unsigned int headroom = virtnet_get_headroom(vi); char *buf; - unsigned long ctx; + void *ctx; int err; unsigned int len, hole; @@ -879,7 +868,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *...

...struct receive_queue *rq, } if (vi->mergeable_rx_bufs) - skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len); + skb = receive_mergeable(dev, vi, rq, buf, ctx, len); else if (vi->big_packets) skb = receive_big(dev, vi, rq, buf, len); else @@ -869,7 +858,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, struct page_frag *alloc_frag = &rq->alloc_frag; unsigned int headroom = virtnet_get_headroom(vi); char *buf; - unsigned long ctx; + void *ctx; int err; unsigned int len, hole; @@ -879,7 +868,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *...

...tions(+), 4 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 3e82311..968eacd 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -604,18 +604,25 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp) return err; } -static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) +static unsigned int get_mergeable_buf_len(struct ewma *avg_pkt_len) { const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + unsigned int len; + + len = hdr_len + clamp_t(unsigned int, ewma_read(avg_pkt_len), + GOOD_PACKET_LEN, PAGE_SIZE -...

...+ write_seqcount_begin(&rq->sysfs_seq); ewma_add(&rq->mrg_avg_pkt_len, head_skb->len); + write_seqcount_end(&rq->sysfs_seq); return head_skb; err_skb: @@ -604,18 +613,29 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp) return err; } -static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp) +static unsigned int get_mergeable_buf_len(struct ewma *avg_pkt_len) { const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + unsigned int len; + + len = hdr_len + clamp_t(unsigned int, ewma_read(avg_pkt_len), + GOOD_PACKET_LEN, PAGE_SIZE -...

skb_page_frag_refill currently permits only order-0 page allocs unless GFP_WAIT is used. Change skb_page_frag_refill to attempt higher-order page allocations whether or not GFP_WAIT is used. If memory cannot be allocated, the allocator will fall back to successively smaller page allocs (down to order-0 page allocs). This change brings skb_page_frag_refill in line with the existing page allocation