Sasha Levin
2012-Aug-28 13:04 UTC
[PATCH v2 1/2] virtio-ring: Use threshold for switching to indirect descriptors
Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will use indirect descriptors even if we have plenty of space in the ring. This means that we take a performance hit at all times due to the overhead of creating indirect descriptors. Instead, use it only after we're below a configurable offset. Signed-off-by: Sasha Levin <levinsasha928 at gmail.com> --- drivers/block/virtio_blk.c | 4 ++++ drivers/char/hw_random/virtio-rng.c | 4 ++++ drivers/char/virtio_console.c | 4 ++++ drivers/net/virtio_net.c | 4 ++++ drivers/virtio/virtio_balloon.c | 4 ++++ drivers/virtio/virtio_ring.c | 21 +++++++++++++++------ include/linux/virtio.h | 1 + net/9p/trans_virtio.c | 4 ++++ 8 files changed, 40 insertions(+), 6 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 2edfb5c..13b8ae9 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -22,6 +22,9 @@ static DEFINE_IDA(vd_index_ida); struct workqueue_struct *virtblk_wq; +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + struct virtio_blk { struct virtio_device *vdev; @@ -735,6 +738,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev) INIT_WORK(&vblk->config_work, virtblk_config_changed_work); vblk->config_enable = true; + vdev->indirect_thresh = indirect_thresh; err = init_vq(vblk); if (err) diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 5708299..02d8421 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -25,6 +25,9 @@ #include <linux/virtio_rng.h> #include <linux/module.h> +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + static struct virtqueue *vq; static unsigned int data_avail; static DECLARE_COMPLETION(have_data); @@ -93,6 +96,7 @@ static int probe_common(struct virtio_device *vdev) int err; /* We expect a single virtqueue. */ + vdev->indirect_thresh = indirect_thresh; vq = virtio_find_single_vq(vdev, random_recv_done, "input"); if (IS_ERR(vq)) return PTR_ERR(vq); diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index e88f843..fc14e7f 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -39,6 +39,9 @@ #include <linux/module.h> #include "../tty/hvc/hvc_console.h" +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + /* * This is a global struct for storing common data for all the devices * this driver handles. @@ -1887,6 +1890,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev) max_nr_ports), &portdev->config.max_nr_ports) == 0) multiport = true; + vdev->indirect_thresh = indirect_thresh; err = init_vqs(portdev); if (err < 0) { diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index cbf8b06..64a8321 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -34,6 +34,9 @@ static bool csum = true, gso = true; module_param(csum, bool, 0444); module_param(gso, bool, 0444); +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + /* FIXME: MTU in config. */ #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) #define GOOD_COPY_LEN 128 @@ -1128,6 +1131,7 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; + vdev->indirect_thresh = indirect_thresh; err = init_vqs(vi); if (err) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 0908e60..fce7347 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -35,6 +35,9 @@ */ #define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + struct virtio_balloon { struct virtio_device *vdev; @@ -356,6 +359,7 @@ static int virtballoon_probe(struct virtio_device *vdev) init_waitqueue_head(&vb->acked); vb->vdev = vdev; vb->need_stats_update = 0; + vdev->indirect_thresh = indirect_thresh; err = init_vqs(vb); if (err) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 5aa43c3..99a64a7 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -87,8 +87,11 @@ struct vring_virtqueue /* Other side has made a mess, don't try any more. */ bool broken; - /* Host supports indirect buffers */ - bool indirect; + /* + * Min. number of free space in the ring to trigger direct + * descriptor use + */ + unsigned int indirect_thresh; /* Host publishes avail event idx */ bool event; @@ -216,9 +219,12 @@ int virtqueue_add_buf(struct virtqueue *_vq, } #endif - /* If the host supports indirect descriptor tables, and we have multiple - * buffers, then go indirect. FIXME: tune this threshold */ - if (vq->indirect && (out + in) > 1 && vq->num_free) { + /* + * If the host supports indirect descriptor tables, and we have multiple + * buffers, then go indirect. + */ + if ((out + in) > 1 && vq->num_free && + (vq->num_free < vq->indirect_thresh)) { head = vring_add_indirect(vq, sg, out, in, gfp); if (likely(head >= 0)) goto add_head; @@ -647,13 +653,16 @@ struct virtqueue *vring_new_virtqueue(unsigned int num, vq->broken = false; vq->last_used_idx = 0; vq->num_added = 0; + vq->indirect_thresh = 0; list_add_tail(&vq->vq.list, &vdev->vqs); #ifdef DEBUG vq->in_use = false; vq->last_add_time_valid = false; #endif - vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) + vq->indirect_thresh = vdev->indirect_thresh; + vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); /* No callback? Tell other side not to bother us. */ diff --git a/include/linux/virtio.h b/include/linux/virtio.h index a1ba8bb..48bc457 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -69,6 +69,7 @@ struct virtio_device { /* Note that this is a Linux set_bit-style bitmap. */ unsigned long features[1]; void *priv; + unsigned int indirect_thresh; }; #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev) diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index 35b8911..fc93962 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -52,6 +52,9 @@ #define VIRTQUEUE_NUM 128 +static unsigned int indirect_thresh; +module_param(indirect_thresh, uint, S_IRUGO); + /* a single mutex to manage channel initialization and attachment */ static DEFINE_MUTEX(virtio_9p_lock); static DECLARE_WAIT_QUEUE_HEAD(vp_wq); @@ -501,6 +504,7 @@ static int p9_virtio_probe(struct virtio_device *vdev) chan->vdev = vdev; /* We expect one virtqueue, for requests. */ + vdev->indirect_thresh = indirect_thresh; chan->vq = virtio_find_single_vq(vdev, req_done, "requests"); if (IS_ERR(chan->vq)) { err = PTR_ERR(chan->vq); -- 1.7.12
Sasha Levin
2012-Aug-28 13:04 UTC
[PATCH v2 2/2] virtio-ring: Allocate indirect buffers from cache when possible
Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will use indirect descriptors and allocate them using a simple kmalloc(). This patch adds a cache which will allow indirect buffers under a configurable size to be allocated from that cache instead. Signed-off-by: Sasha Levin <levinsasha928 at gmail.com> --- Changes in v2: - Free correctly indirect buffers. drivers/block/virtio_blk.c | 4 ++++ drivers/char/hw_random/virtio-rng.c | 4 ++++ drivers/char/virtio_console.c | 4 ++++ drivers/net/virtio_net.c | 4 ++++ drivers/virtio/virtio_balloon.c | 4 ++++ drivers/virtio/virtio_ring.c | 28 ++++++++++++++++++++++++---- include/linux/virtio.h | 1 + net/9p/trans_virtio.c | 5 +++++ 8 files changed, 50 insertions(+), 4 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 13b8ae9..7f670af 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -25,6 +25,9 @@ struct workqueue_struct *virtblk_wq; static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + struct virtio_blk { struct virtio_device *vdev; @@ -739,6 +742,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev) INIT_WORK(&vblk->config_work, virtblk_config_changed_work); vblk->config_enable = true; vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; err = init_vq(vblk); if (err) diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 02d8421..8475ece 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -28,6 +28,9 @@ static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + static struct virtqueue *vq; static unsigned int data_avail; static DECLARE_COMPLETION(have_data); @@ -97,6 +100,7 @@ static int probe_common(struct virtio_device *vdev) /* We expect a single virtqueue. */ vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; vq = virtio_find_single_vq(vdev, random_recv_done, "input"); if (IS_ERR(vq)) return PTR_ERR(vq); diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index fc14e7f..c6f24a7 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -42,6 +42,9 @@ static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + /* * This is a global struct for storing common data for all the devices * this driver handles. @@ -1891,6 +1894,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev) &portdev->config.max_nr_ports) == 0) multiport = true; vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; err = init_vqs(portdev); if (err < 0) { diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 64a8321..c091efd 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -37,6 +37,9 @@ module_param(gso, bool, 0444); static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + /* FIXME: MTU in config. */ #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) #define GOOD_COPY_LEN 128 @@ -1132,6 +1135,7 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; err = init_vqs(vi); if (err) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index fce7347..ccf87db 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -38,6 +38,9 @@ static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + struct virtio_balloon { struct virtio_device *vdev; @@ -360,6 +363,7 @@ static int virtballoon_probe(struct virtio_device *vdev) vb->vdev = vdev; vb->need_stats_update = 0; vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; err = init_vqs(vb); if (err) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 99a64a7..e8b9c54 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -93,6 +93,10 @@ struct vring_virtqueue */ unsigned int indirect_thresh; + /* Buffers below this size will be allocated from cache */ + unsigned int indirect_alloc_thresh; + struct kmem_cache *indirect_cache; + /* Host publishes avail event idx */ bool event; @@ -135,7 +139,10 @@ static int vring_add_indirect(struct vring_virtqueue *vq, unsigned head; int i; - desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); + if ((out + in) <= vq->indirect_alloc_thresh) + desc = kmem_cache_alloc(vq->indirect_cache, gfp); + else + desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); if (!desc) return -ENOMEM; @@ -384,8 +391,13 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head) i = head; /* Free the indirect table */ - if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) - kfree(phys_to_virt(vq->vring.desc[i].addr)); + if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) { + if (vq->vring.desc[i].len > vq->indirect_alloc_thresh) + kfree(phys_to_virt(vq->vring.desc[i].addr)); + else + kmem_cache_free(vq->indirect_cache, + phys_to_virt(vq->vring.desc[i].addr)); + } while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { i = vq->vring.desc[i].next; @@ -654,14 +666,20 @@ struct virtqueue *vring_new_virtqueue(unsigned int num, vq->last_used_idx = 0; vq->num_added = 0; vq->indirect_thresh = 0; + vq->indirect_alloc_thresh = 0; + vq->indirect_cache = NULL; list_add_tail(&vq->vq.list, &vdev->vqs); #ifdef DEBUG vq->in_use = false; vq->last_add_time_valid = false; #endif - if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) { vq->indirect_thresh = vdev->indirect_thresh; + vq->indirect_alloc_thresh = vdev->indirect_alloc_thresh; + if (vq->indirect_alloc_thresh) + vq->indirect_cache = KMEM_CACHE(vring_desc[vq->indirect_alloc_thresh], 0); + } vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); @@ -685,6 +703,8 @@ EXPORT_SYMBOL_GPL(vring_new_virtqueue); void vring_del_virtqueue(struct virtqueue *vq) { list_del(&vq->list); + if (to_vvq(vq)->indirect_cache) + kmem_cache_destroy(to_vvq(vq)->indirect_cache); kfree(to_vvq(vq)); } EXPORT_SYMBOL_GPL(vring_del_virtqueue); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 48bc457..3261c02 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -70,6 +70,7 @@ struct virtio_device { unsigned long features[1]; void *priv; unsigned int indirect_thresh; + unsigned int indirect_alloc_thresh; }; #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev) diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index fc93962..3044c86 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -55,6 +55,9 @@ static unsigned int indirect_thresh; module_param(indirect_thresh, uint, S_IRUGO); +static unsigned int indirect_alloc_thresh; +module_param(indirect_alloc_thresh, uint, S_IRUGO); + /* a single mutex to manage channel initialization and attachment */ static DEFINE_MUTEX(virtio_9p_lock); static DECLARE_WAIT_QUEUE_HEAD(vp_wq); @@ -505,6 +508,8 @@ static int p9_virtio_probe(struct virtio_device *vdev) /* We expect one virtqueue, for requests. */ vdev->indirect_thresh = indirect_thresh; + vdev->indirect_alloc_thresh = indirect_alloc_thresh; + chan->vq = virtio_find_single_vq(vdev, req_done, "requests"); if (IS_ERR(chan->vq)) { err = PTR_ERR(chan->vq); -- 1.7.12
Michael S. Tsirkin
2012-Aug-28 13:20 UTC
[PATCH v2 1/2] virtio-ring: Use threshold for switching to indirect descriptors
On Tue, Aug 28, 2012 at 03:04:02PM +0200, Sasha Levin wrote:> Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will use indirect > descriptors even if we have plenty of space in the ring. This means that > we take a performance hit at all times due to the overhead of creating > indirect descriptors. > > Instead, use it only after we're below a configurable offset. > > Signed-off-by: Sasha Levin <levinsasha928 at gmail.com>I imagine this helps performance? Any numbers?> --- > drivers/block/virtio_blk.c | 4 ++++ > drivers/char/hw_random/virtio-rng.c | 4 ++++ > drivers/char/virtio_console.c | 4 ++++ > drivers/net/virtio_net.c | 4 ++++ > drivers/virtio/virtio_balloon.c | 4 ++++ > drivers/virtio/virtio_ring.c | 21 +++++++++++++++------ > include/linux/virtio.h | 1 + > net/9p/trans_virtio.c | 4 ++++ > 8 files changed, 40 insertions(+), 6 deletions(-) > > diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c > index 2edfb5c..13b8ae9 100644 > --- a/drivers/block/virtio_blk.c > +++ b/drivers/block/virtio_blk.c > @@ -22,6 +22,9 @@ static DEFINE_IDA(vd_index_ida); > > struct workqueue_struct *virtblk_wq; > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > struct virtio_blk > { > struct virtio_device *vdev; > @@ -735,6 +738,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev) > > INIT_WORK(&vblk->config_work, virtblk_config_changed_work); > vblk->config_enable = true; > + vdev->indirect_thresh = indirect_thresh; > > err = init_vq(vblk); > if (err) > diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c > index 5708299..02d8421 100644 > --- a/drivers/char/hw_random/virtio-rng.c > +++ b/drivers/char/hw_random/virtio-rng.c > @@ -25,6 +25,9 @@ > #include <linux/virtio_rng.h> > #include <linux/module.h> > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > static struct virtqueue *vq; > static unsigned int data_avail; > static DECLARE_COMPLETION(have_data); > @@ -93,6 +96,7 @@ static int probe_common(struct virtio_device *vdev) > int err; > > /* We expect a single virtqueue. */ > + vdev->indirect_thresh = indirect_thresh; > vq = virtio_find_single_vq(vdev, random_recv_done, "input"); > if (IS_ERR(vq)) > return PTR_ERR(vq); > diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c > index e88f843..fc14e7f 100644 > --- a/drivers/char/virtio_console.c > +++ b/drivers/char/virtio_console.c > @@ -39,6 +39,9 @@ > #include <linux/module.h> > #include "../tty/hvc/hvc_console.h" > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > /* > * This is a global struct for storing common data for all the devices > * this driver handles. > @@ -1887,6 +1890,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev) > max_nr_ports), > &portdev->config.max_nr_ports) == 0) > multiport = true; > + vdev->indirect_thresh = indirect_thresh; > > err = init_vqs(portdev); > if (err < 0) { > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index cbf8b06..64a8321 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -34,6 +34,9 @@ static bool csum = true, gso = true; > module_param(csum, bool, 0444); > module_param(gso, bool, 0444); > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > /* FIXME: MTU in config. */ > #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > #define GOOD_COPY_LEN 128 > @@ -1128,6 +1131,7 @@ static int virtnet_probe(struct virtio_device *vdev) > > if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) > vi->mergeable_rx_bufs = true; > + vdev->indirect_thresh = indirect_thresh; > > err = init_vqs(vi); > if (err) > diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c > index 0908e60..fce7347 100644 > --- a/drivers/virtio/virtio_balloon.c > +++ b/drivers/virtio/virtio_balloon.c > @@ -35,6 +35,9 @@ > */ > #define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > struct virtio_balloon > { > struct virtio_device *vdev; > @@ -356,6 +359,7 @@ static int virtballoon_probe(struct virtio_device *vdev) > init_waitqueue_head(&vb->acked); > vb->vdev = vdev; > vb->need_stats_update = 0; > + vdev->indirect_thresh = indirect_thresh; > > err = init_vqs(vb); > if (err) > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c > index 5aa43c3..99a64a7 100644 > --- a/drivers/virtio/virtio_ring.c > +++ b/drivers/virtio/virtio_ring.c > @@ -87,8 +87,11 @@ struct vring_virtqueue > /* Other side has made a mess, don't try any more. */ > bool broken; > > - /* Host supports indirect buffers */ > - bool indirect; > + /* > + * Min. number of free space in the ring to trigger direct > + * descriptor use > + */ > + unsigned int indirect_thresh; > > /* Host publishes avail event idx */ > bool event; > @@ -216,9 +219,12 @@ int virtqueue_add_buf(struct virtqueue *_vq, > } > #endif > > - /* If the host supports indirect descriptor tables, and we have multiple > - * buffers, then go indirect. FIXME: tune this threshold */ > - if (vq->indirect && (out + in) > 1 && vq->num_free) { > + /* > + * If the host supports indirect descriptor tables, and we have multiple > + * buffers, then go indirect. > + */ > + if ((out + in) > 1 && vq->num_free && > + (vq->num_free < vq->indirect_thresh)) { > head = vring_add_indirect(vq, sg, out, in, gfp); > if (likely(head >= 0)) > goto add_head; > @@ -647,13 +653,16 @@ struct virtqueue *vring_new_virtqueue(unsigned int num, > vq->broken = false; > vq->last_used_idx = 0; > vq->num_added = 0; > + vq->indirect_thresh = 0; > list_add_tail(&vq->vq.list, &vdev->vqs); > #ifdef DEBUG > vq->in_use = false; > vq->last_add_time_valid = false; > #endif > > - vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); > + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) > + vq->indirect_thresh = vdev->indirect_thresh; > + > vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); > > /* No callback? Tell other side not to bother us. */ > diff --git a/include/linux/virtio.h b/include/linux/virtio.h > index a1ba8bb..48bc457 100644 > --- a/include/linux/virtio.h > +++ b/include/linux/virtio.h > @@ -69,6 +69,7 @@ struct virtio_device { > /* Note that this is a Linux set_bit-style bitmap. */ > unsigned long features[1]; > void *priv; > + unsigned int indirect_thresh; > }; > > #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev) > diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c > index 35b8911..fc93962 100644 > --- a/net/9p/trans_virtio.c > +++ b/net/9p/trans_virtio.c > @@ -52,6 +52,9 @@ > > #define VIRTQUEUE_NUM 128 > > +static unsigned int indirect_thresh; > +module_param(indirect_thresh, uint, S_IRUGO); > + > /* a single mutex to manage channel initialization and attachment */ > static DEFINE_MUTEX(virtio_9p_lock); > static DECLARE_WAIT_QUEUE_HEAD(vp_wq); > @@ -501,6 +504,7 @@ static int p9_virtio_probe(struct virtio_device *vdev) > chan->vdev = vdev; > > /* We expect one virtqueue, for requests. */ > + vdev->indirect_thresh = indirect_thresh; > chan->vq = virtio_find_single_vq(vdev, req_done, "requests"); > if (IS_ERR(chan->vq)) { > err = PTR_ERR(chan->vq); > -- > 1.7.12
Michael S. Tsirkin
2012-Aug-28 13:20 UTC
[PATCH v2 2/2] virtio-ring: Allocate indirect buffers from cache when possible
On Tue, Aug 28, 2012 at 03:04:03PM +0200, Sasha Levin wrote:> Currently if VIRTIO_RING_F_INDIRECT_DESC is enabled we will > use indirect descriptors and allocate them using a simple > kmalloc(). > > This patch adds a cache which will allow indirect buffers under > a configurable size to be allocated from that cache instead. > > Signed-off-by: Sasha Levin <levinsasha928 at gmail.com>I imagine this helps performance? Any numbers?> --- > > Changes in v2: > > - Free correctly indirect buffers. > > drivers/block/virtio_blk.c | 4 ++++ > drivers/char/hw_random/virtio-rng.c | 4 ++++ > drivers/char/virtio_console.c | 4 ++++ > drivers/net/virtio_net.c | 4 ++++ > drivers/virtio/virtio_balloon.c | 4 ++++ > drivers/virtio/virtio_ring.c | 28 ++++++++++++++++++++++++---- > include/linux/virtio.h | 1 + > net/9p/trans_virtio.c | 5 +++++ > 8 files changed, 50 insertions(+), 4 deletions(-) > > diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c > index 13b8ae9..7f670af 100644 > --- a/drivers/block/virtio_blk.c > +++ b/drivers/block/virtio_blk.c > @@ -25,6 +25,9 @@ struct workqueue_struct *virtblk_wq; > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > struct virtio_blk > { > struct virtio_device *vdev; > @@ -739,6 +742,7 @@ static int __devinit virtblk_probe(struct virtio_device *vdev) > INIT_WORK(&vblk->config_work, virtblk_config_changed_work); > vblk->config_enable = true; > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > > err = init_vq(vblk); > if (err) > diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c > index 02d8421..8475ece 100644 > --- a/drivers/char/hw_random/virtio-rng.c > +++ b/drivers/char/hw_random/virtio-rng.c > @@ -28,6 +28,9 @@ > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > static struct virtqueue *vq; > static unsigned int data_avail; > static DECLARE_COMPLETION(have_data); > @@ -97,6 +100,7 @@ static int probe_common(struct virtio_device *vdev) > > /* We expect a single virtqueue. */ > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > vq = virtio_find_single_vq(vdev, random_recv_done, "input"); > if (IS_ERR(vq)) > return PTR_ERR(vq); > diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c > index fc14e7f..c6f24a7 100644 > --- a/drivers/char/virtio_console.c > +++ b/drivers/char/virtio_console.c > @@ -42,6 +42,9 @@ > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > /* > * This is a global struct for storing common data for all the devices > * this driver handles. > @@ -1891,6 +1894,7 @@ static int __devinit virtcons_probe(struct virtio_device *vdev) > &portdev->config.max_nr_ports) == 0) > multiport = true; > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > > err = init_vqs(portdev); > if (err < 0) { > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 64a8321..c091efd 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -37,6 +37,9 @@ module_param(gso, bool, 0444); > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > /* FIXME: MTU in config. */ > #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > #define GOOD_COPY_LEN 128 > @@ -1132,6 +1135,7 @@ static int virtnet_probe(struct virtio_device *vdev) > if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) > vi->mergeable_rx_bufs = true; > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > > err = init_vqs(vi); > if (err) > diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c > index fce7347..ccf87db 100644 > --- a/drivers/virtio/virtio_balloon.c > +++ b/drivers/virtio/virtio_balloon.c > @@ -38,6 +38,9 @@ > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > struct virtio_balloon > { > struct virtio_device *vdev; > @@ -360,6 +363,7 @@ static int virtballoon_probe(struct virtio_device *vdev) > vb->vdev = vdev; > vb->need_stats_update = 0; > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > > err = init_vqs(vb); > if (err) > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c > index 99a64a7..e8b9c54 100644 > --- a/drivers/virtio/virtio_ring.c > +++ b/drivers/virtio/virtio_ring.c > @@ -93,6 +93,10 @@ struct vring_virtqueue > */ > unsigned int indirect_thresh; > > + /* Buffers below this size will be allocated from cache */ > + unsigned int indirect_alloc_thresh; > + struct kmem_cache *indirect_cache; > + > /* Host publishes avail event idx */ > bool event; > > @@ -135,7 +139,10 @@ static int vring_add_indirect(struct vring_virtqueue *vq, > unsigned head; > int i; > > - desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); > + if ((out + in) <= vq->indirect_alloc_thresh) > + desc = kmem_cache_alloc(vq->indirect_cache, gfp); > + else > + desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); > if (!desc) > return -ENOMEM; > > @@ -384,8 +391,13 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head) > i = head; > > /* Free the indirect table */ > - if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) > - kfree(phys_to_virt(vq->vring.desc[i].addr)); > + if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) { > + if (vq->vring.desc[i].len > vq->indirect_alloc_thresh) > + kfree(phys_to_virt(vq->vring.desc[i].addr)); > + else > + kmem_cache_free(vq->indirect_cache, > + phys_to_virt(vq->vring.desc[i].addr)); > + } > > while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { > i = vq->vring.desc[i].next; > @@ -654,14 +666,20 @@ struct virtqueue *vring_new_virtqueue(unsigned int num, > vq->last_used_idx = 0; > vq->num_added = 0; > vq->indirect_thresh = 0; > + vq->indirect_alloc_thresh = 0; > + vq->indirect_cache = NULL; > list_add_tail(&vq->vq.list, &vdev->vqs); > #ifdef DEBUG > vq->in_use = false; > vq->last_add_time_valid = false; > #endif > > - if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) > + if (virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) { > vq->indirect_thresh = vdev->indirect_thresh; > + vq->indirect_alloc_thresh = vdev->indirect_alloc_thresh; > + if (vq->indirect_alloc_thresh) > + vq->indirect_cache = KMEM_CACHE(vring_desc[vq->indirect_alloc_thresh], 0); > + } > > vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); > > @@ -685,6 +703,8 @@ EXPORT_SYMBOL_GPL(vring_new_virtqueue); > void vring_del_virtqueue(struct virtqueue *vq) > { > list_del(&vq->list); > + if (to_vvq(vq)->indirect_cache) > + kmem_cache_destroy(to_vvq(vq)->indirect_cache); > kfree(to_vvq(vq)); > } > EXPORT_SYMBOL_GPL(vring_del_virtqueue); > diff --git a/include/linux/virtio.h b/include/linux/virtio.h > index 48bc457..3261c02 100644 > --- a/include/linux/virtio.h > +++ b/include/linux/virtio.h > @@ -70,6 +70,7 @@ struct virtio_device { > unsigned long features[1]; > void *priv; > unsigned int indirect_thresh; > + unsigned int indirect_alloc_thresh; > }; > > #define dev_to_virtio(dev) container_of(dev, struct virtio_device, dev) > diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c > index fc93962..3044c86 100644 > --- a/net/9p/trans_virtio.c > +++ b/net/9p/trans_virtio.c > @@ -55,6 +55,9 @@ > static unsigned int indirect_thresh; > module_param(indirect_thresh, uint, S_IRUGO); > > +static unsigned int indirect_alloc_thresh; > +module_param(indirect_alloc_thresh, uint, S_IRUGO); > + > /* a single mutex to manage channel initialization and attachment */ > static DEFINE_MUTEX(virtio_9p_lock); > static DECLARE_WAIT_QUEUE_HEAD(vp_wq); > @@ -505,6 +508,8 @@ static int p9_virtio_probe(struct virtio_device *vdev) > > /* We expect one virtqueue, for requests. */ > vdev->indirect_thresh = indirect_thresh; > + vdev->indirect_alloc_thresh = indirect_alloc_thresh; > + > chan->vq = virtio_find_single_vq(vdev, req_done, "requests"); > if (IS_ERR(chan->vq)) { > err = PTR_ERR(chan->vq); > -- > 1.7.12
Possibly Parallel Threads
- [PATCH v2 1/2] virtio-ring: Use threshold for switching to indirect descriptors
- [RFC 1/2] virtio-ring: Use threshold for switching to indirect descriptors
- [RFC 1/2] virtio-ring: Use threshold for switching to indirect descriptors
- [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors
- [PATCH v3 1/2] virtio-ring: Use threshold for switching to indirect descriptors