search for: l1_cache_bytes

...r size is up to 1 page so we need to align to + * square root of page size to ensure we reserve enough bits to encode the true + * size. + */ +#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) + /* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256) +#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ + 1U << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) #define VIRTNET_DRIVER_VERSION "1.0.0" -- MST

...r size is up to 1 page so we need to align to + * square root of page size to ensure we reserve enough bits to encode the true + * size. + */ +#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) + /* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256) +#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ + 1U << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) #define VIRTNET_DRIVER_VERSION "1.0.0" -- MST

...minimum buffer alignment of only 64, and has a maximum theoretical buffer size of aligned GOOD_PACKET_LEN + (BUF_ALIGN - 1) * BUF_ALIGN, which is at least 1536 + 63 * 64 = 5568. On x86, we already use a 64 byte alignment, and this code supports all current buffer sizes, from 1536 to PAGE_SIZE. #if L1_CACHE_BYTES < 64 #define MERGEABLE_BUFFER_ALIGN 64 #define MERGEABLE_BUFFER_SHIFT 6 #else #define MERGEABLE_BUFFER_ALIGN L1_CACHE_BYTES #define MERGEABLE_BUFFER_SHIFT L1_CACHE_SHIFT #endif #define MERGEABLE_BUFFER_MIN ALIGN(GOOD_PACKET_LEN + sizeof(virtio_net_hdr_mrg_rbuf)...

...minimum buffer alignment of only 64, and has a maximum theoretical buffer size of aligned GOOD_PACKET_LEN + (BUF_ALIGN - 1) * BUF_ALIGN, which is at least 1536 + 63 * 64 = 5568. On x86, we already use a 64 byte alignment, and this code supports all current buffer sizes, from 1536 to PAGE_SIZE. #if L1_CACHE_BYTES < 64 #define MERGEABLE_BUFFER_ALIGN 64 #define MERGEABLE_BUFFER_SHIFT 6 #else #define MERGEABLE_BUFFER_ALIGN L1_CACHE_BYTES #define MERGEABLE_BUFFER_SHIFT L1_CACHE_SHIFT #endif #define MERGEABLE_BUFFER_MIN ALIGN(GOOD_PACKET_LEN + sizeof(virtio_net_hdr_mrg_rbuf)...

...r size is up to 1 page so we need to align to - * square root of page size to ensure we reserve enough bits to encode the true - * size. - */ -#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) - -/* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ - 1 << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) - #define VIRTNET_DRIVER_VERSION "1.0.0" struct virtnet_stats { @@ -849,7 +838,7 @@ static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len) len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_l...

...r size is up to 1 page so we need to align to - * square root of page size to ensure we reserve enough bits to encode the true - * size. - */ -#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) - -/* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ - 1 << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) - #define VIRTNET_DRIVER_VERSION "1.0.0" struct virtnet_stats { @@ -849,7 +838,7 @@ static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len) len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_l...

...PAGE_SIZE. I really think it's best to start with 256 alignment. A bit simpler, better than what we had, and will let us go above PAGE_SIZE long term. The optimization shrinking alignment to 64 can be done on top if we see a work-load that's improved by it, which I doubt. > > #if L1_CACHE_BYTES < 64 > #define MERGEABLE_BUFFER_ALIGN 64 > #define MERGEABLE_BUFFER_SHIFT 6 > #else > #define MERGEABLE_BUFFER_ALIGN L1_CACHE_BYTES > #define MERGEABLE_BUFFER_SHIFT L1_CACHE_SHIFT > #endif > #define MERGEABLE_BUFFER_MIN ALIGN(GOOD_PACKET_LEN + >...

...2,7 +842,7 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq, unsigned int len; len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), - rq->min_buf_len - hdr_len, PAGE_SIZE - hdr_len); + rq->min_buf_len, PAGE_SIZE - hdr_len); return ALIGN(len, L1_CACHE_BYTES); } @@ -2039,7 +2039,8 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); - return max(min_buf_len, hdr_len); + return max(max(min...

...2,7 +842,7 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq, unsigned int len; len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), - rq->min_buf_len - hdr_len, PAGE_SIZE - hdr_len); + rq->min_buf_len, PAGE_SIZE - hdr_len); return ALIGN(len, L1_CACHE_BYTES); } @@ -2039,7 +2039,8 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); - return max(min_buf_len, hdr_len); + return max(max(min...

Sorry, forgot to mention - if we want to explore combining the buffer address and truesize into a single void *, we could also exploit the fact that our size ranges from aligned GOOD_PACKET_LEN to PAGE_SIZE, and potentially encode fewer values for truesize (and require a smaller alignment than 256). The prior e-mails discussion of 256 byte alignment with 256 values is just one potential design

Sorry, forgot to mention - if we want to explore combining the buffer address and truesize into a single void *, we could also exploit the fact that our size ranges from aligned GOOD_PACKET_LEN to PAGE_SIZE, and potentially encode fewer values for truesize (and require a smaller alignment than 256). The prior e-mails discussion of 256 byte alignment with 256 values is just one potential design

...r size is up to 1 page so we need to align to + * square root of page size to ensure we reserve enough bits to encode the true + * size. + */ +#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) + /* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256) +#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ + 1 << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) #define VIRTNET_DRIVER_VERSION "1.0.0" -- MST

...r size is up to 1 page so we need to align to + * square root of page size to ensure we reserve enough bits to encode the true + * size. + */ +#define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2) + /* Minimum alignment for mergeable packet buffers. */ -#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256) +#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \ + 1 << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT) #define VIRTNET_DRIVER_VERSION "1.0.0" -- MST

...r25=ar.unat /* M */ >+ mov r26=ar.pfs /* I */ >+ mov r28=cr.iip /* M */ >+ cover /* B (or nothing) */ >+ ;; >+ mov r1=sp >+ ;; >+ invala /* M */ >+ mov r30=cr.ifs >+ ;; >+ addl r1=-VMM_PT_REGS_SIZE,r1 >+ ;; >+ adds r17=2*L1_CACHE_BYTES,r1 /* really: biggest cache-line >size */ >+ adds r16=PT(CR_IPSR),r1 >+ ;; >+ lfetch.fault.excl.nt1 [r17],L1_CACHE_BYTES >+ st8 [r16]=r29 /* save cr.ipsr */ >+ ;; >+ lfetch.fault.excl.nt1 [r17] >+ mov r29=b0 >+ ;; >+ adds r16=PT(R8),r1 /*...

...r25=ar.unat /* M */ >+ mov r26=ar.pfs /* I */ >+ mov r28=cr.iip /* M */ >+ cover /* B (or nothing) */ >+ ;; >+ mov r1=sp >+ ;; >+ invala /* M */ >+ mov r30=cr.ifs >+ ;; >+ addl r1=-VMM_PT_REGS_SIZE,r1 >+ ;; >+ adds r17=2*L1_CACHE_BYTES,r1 /* really: biggest cache-line >size */ >+ adds r16=PT(CR_IPSR),r1 >+ ;; >+ lfetch.fault.excl.nt1 [r17],L1_CACHE_BYTES >+ st8 [r16]=r29 /* save cr.ipsr */ >+ ;; >+ lfetch.fault.excl.nt1 [r17] >+ mov r29=b0 >+ ;; >+ adds r16=PT(R8),r1 /*...

...bool, 0444); > > /* FIXME: MTU in config. */ > #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > -#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \ > - sizeof(struct virtio_net_hdr_mrg_rxbuf), \ > - L1_CACHE_BYTES)) > #define GOOD_COPY_LEN 128 > > +/* Weight used for the RX packet size EWMA. The average packet size is used to > + * determine the packet buffer size when refilling RX rings. As the entire RX > + * ring may be refilled at once, the weight is chosen so that the EWMA will be >...

...bool, 0444); > > /* FIXME: MTU in config. */ > #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > -#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \ > - sizeof(struct virtio_net_hdr_mrg_rxbuf), \ > - L1_CACHE_BYTES)) > #define GOOD_COPY_LEN 128 > > +/* Weight used for the RX packet size EWMA. The average packet size is used to > + * determine the packet buffer size when refilling RX rings. As the entire RX > + * ring may be refilled at once, the weight is chosen so that the EWMA will be >...

...n config. */ -#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) +#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) +#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \ + sizeof(struct virtio_net_hdr_mrg_rxbuf), \ + L1_CACHE_BYTES)) #define GOOD_COPY_LEN 128 #define VIRTNET_DRIVER_VERSION "1.0.0" @@ -314,10 +317,10 @@ static int receive_mergeable(struct receive_queue *rq, struct sk_buff *head_skb) head_skb->dev->stats.rx_length_errors++; return -EINVAL; } - if (unlikely(len > MAX_PACKET_LEN...

...n config. */ -#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) +#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) +#define MERGE_BUFFER_LEN (ALIGN(GOOD_PACKET_LEN + \ + sizeof(struct virtio_net_hdr_mrg_rxbuf), \ + L1_CACHE_BYTES)) #define GOOD_COPY_LEN 128 #define VIRTNET_DRIVER_VERSION "1.0.0" @@ -314,10 +317,10 @@ static int receive_mergeable(struct receive_queue *rq, struct sk_buff *head_skb) head_skb->dev->stats.rx_length_errors++; return -EINVAL; } - if (unlikely(len > MAX_PACKET_LEN...

...en *avg_pkt_len) { const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); unsigned int len; len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), - GOOD_PACKET_LEN, PAGE_SIZE - hdr_len); + rq->min_buf_len - hdr_len, PAGE_SIZE - hdr_len); return ALIGN(len, L1_CACHE_BYTES); } @@ -914,7 +918,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, int err; unsigned int len, hole; - len = get_mergeable_buf_len(&rq->mrg_avg_pkt_len); + len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len); if (unlikely(!skb_page_frag_refill(len + headroom...