Jason Wang
2022-Nov-14 07:58 UTC
[PATCH 3/6] vduse: Add sysfs interface for irq affinity setup
On Mon, Nov 14, 2022 at 3:16 PM Xie Yongji <xieyongji at bytedance.com> wrote:> > Add sysfs interface for each vduse virtqueue to setup > irq affinity. This would be useful for performance > tuning, e.g., mitigate the virtqueue lock contention > in virtio block driver.Do we have any perforamnce numbers for this? Btw, I wonder if irq is the best for the name since we actually don't use IRQ at all. I guess using "callback" might be better? Thanks> > Signed-off-by: Xie Yongji <xieyongji at bytedance.com> > --- > drivers/vdpa/vdpa_user/vduse_dev.c | 113 ++++++++++++++++++++++++++--- > 1 file changed, 102 insertions(+), 11 deletions(-) > > diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c > index 9303942c2e64..3a0922fa7eb2 100644 > --- a/drivers/vdpa/vdpa_user/vduse_dev.c > +++ b/drivers/vdpa/vdpa_user/vduse_dev.c > @@ -57,6 +57,7 @@ struct vduse_virtqueue { > struct vdpa_callback cb; > struct work_struct inject; > struct work_struct kick; > + struct kobject kobj; > int irq_affinity; > }; > > @@ -1347,6 +1348,88 @@ static const struct file_operations vduse_dev_fops = { > .llseek = noop_llseek, > }; > > +static ssize_t irq_affinity_show(struct vduse_virtqueue *vq, char *buf) > +{ > + return sprintf(buf, "%d\n", vq->irq_affinity); > +} > + > +static ssize_t irq_affinity_store(struct vduse_virtqueue *vq, > + const char *buf, size_t count) > +{ > + int val; > + > + if (kstrtoint(buf, 0, &val) < 0) > + return -EINVAL; > + > + if (!(val == -1 || (val <= nr_cpu_ids && val >= 0 && cpu_online(val)))) > + return -EINVAL; > + > + vq->irq_affinity = val; > + > + return count; > +} > + > +struct vq_sysfs_entry { > + struct attribute attr; > + ssize_t (*show)(struct vduse_virtqueue *vq, char *buf); > + ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf, > + size_t count); > +}; > + > +static struct vq_sysfs_entry irq_affinity_attr = __ATTR_RW(irq_affinity); > + > +static struct attribute *vq_attrs[] = { > + &irq_affinity_attr.attr, > + NULL, > +}; > +ATTRIBUTE_GROUPS(vq); > + > +static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, > + char *buf) > +{ > + struct vduse_virtqueue *vq = container_of(kobj, > + struct vduse_virtqueue, kobj); > + struct vq_sysfs_entry *entry = container_of(attr, > + struct vq_sysfs_entry, attr); > + > + if (!entry->show) > + return -EIO; > + > + return entry->show(vq, buf); > +} > + > +static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, > + const char *buf, size_t count) > +{ > + struct vduse_virtqueue *vq = container_of(kobj, > + struct vduse_virtqueue, kobj); > + struct vq_sysfs_entry *entry = container_of(attr, > + struct vq_sysfs_entry, attr); > + > + if (!entry->store) > + return -EIO; > + > + return entry->store(vq, buf, count); > +} > + > +static const struct sysfs_ops vq_sysfs_ops = { > + .show = vq_attr_show, > + .store = vq_attr_store, > +}; > + > +static void vq_release(struct kobject *kobj) > +{ > + struct vduse_virtqueue *vq = container_of(kobj, > + struct vduse_virtqueue, kobj); > + kfree(vq); > +} > + > +static struct kobj_type vq_type = { > + .release = vq_release, > + .sysfs_ops = &vq_sysfs_ops, > + .default_groups = vq_groups, > +}; > + > static void vduse_dev_deinit_vqs(struct vduse_dev *dev) > { > int i; > @@ -1355,13 +1438,13 @@ static void vduse_dev_deinit_vqs(struct vduse_dev *dev) > return; > > for (i = 0; i < dev->vq_num; i++) > - kfree(dev->vqs[i]); > + kobject_put(&dev->vqs[i]->kobj); > kfree(dev->vqs); > } > > static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) > { > - int i; > + int ret, i; > > dev->vq_align = vq_align; > dev->vq_num = vq_num; > @@ -1371,8 +1454,10 @@ static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) > > for (i = 0; i < vq_num; i++) { > dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL); > - if (!dev->vqs[i]) > + if (!dev->vqs[i]) { > + ret = -ENOMEM; > goto err; > + } > > dev->vqs[i]->index = i; > dev->vqs[i]->irq_affinity = -1; > @@ -1380,15 +1465,20 @@ static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) > INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); > spin_lock_init(&dev->vqs[i]->kick_lock); > spin_lock_init(&dev->vqs[i]->irq_lock); > + kobject_init(&dev->vqs[i]->kobj, &vq_type); > + ret = kobject_add(&dev->vqs[i]->kobj, > + &dev->dev->kobj, "vq%d", i); > + if (ret) > + goto err; > } > > return 0; > err: > while (i--) > - kfree(dev->vqs[i]); > + kobject_put(&dev->vqs[i]->kobj); > kfree(dev->vqs); > dev->vqs = NULL; > - return -ENOMEM; > + return ret; > } > > static struct vduse_dev *vduse_dev_create(void) > @@ -1563,10 +1653,6 @@ static int vduse_create_dev(struct vduse_dev_config *config, > dev->config = config_buf; > dev->config_size = config->config_size; > > - ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); > - if (ret) > - goto err_vqs; > - > ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); > if (ret < 0) > goto err_idr; > @@ -1580,14 +1666,19 @@ static int vduse_create_dev(struct vduse_dev_config *config, > ret = PTR_ERR(dev->dev); > goto err_dev; > } > + > + ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); > + if (ret) > + goto err_vqs; > + > __module_get(THIS_MODULE); > > return 0; > +err_vqs: > + device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); > err_dev: > idr_remove(&vduse_idr, dev->minor); > err_idr: > - vduse_dev_deinit_vqs(dev); > -err_vqs: > vduse_domain_destroy(dev->domain); > err_domain: > kfree(dev->name); > -- > 2.20.1 >