search for: vq_iotlb_prefetch

...3 files changed, 271 insertions(+), 7 deletions(-) diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 36f3d0f49e60..0b4b3deab5aa 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -971,7 +971,7 @@ static void handle_tx(struct vhost_net *net) if (!sock) goto out; - if (!vq_iotlb_prefetch(vq)) + if (!vq_meta_prefetch(vq)) goto out; vhost_disable_notify(&net->dev, vq); @@ -1140,7 +1140,7 @@ static void handle_rx(struct vhost_net *net) if (!sock) goto out; - if (!vq_iotlb_prefetch(vq)) + if (!vq_meta_prefetch(vq)) goto out; vhost_disable_notify(&net-&gt...

..._event, sizeof(*driver_event)) && + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); } static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, @@ -1186,14 +1191,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, return true; } -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) +{ + int num = vq->num; + + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && + iotlb_access_ok(vq, VHOST_ACCESS_...

...r_event, sizeof(*driver_event)) && + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); } static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, @@ -1222,14 +1227,27 @@ static int iotlb_access_ok(struct vhost_virtqueue *vq, return true; } -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) +{ + int num = vq->num; + + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && + iotlb_access_ok(vq, VHOST_ACCESS_...

..._event, sizeof(*driver_event)) && + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); } static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, @@ -1193,14 +1198,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, return true; } -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) +{ + int num = vq->num; + + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && + iotlb_access_ok(vq, VHOST_ACCESS_...

..._event, sizeof(*driver_event)) && + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); } static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, @@ -1190,14 +1195,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, return true; } -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) +{ + int num = vq->num; + + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && + iotlb_access_ok(vq, VHOST_ACCESS_...

...; + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); > } > > static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, > @@ -1190,14 +1195,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, > return true; > } > > -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) > +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) > +{ > + int num = vq->num; > + > + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, > + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && > +...

...; + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); > } > > static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, > @@ -1190,14 +1195,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, > return true; > } > > -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) > +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) > +{ > + int num = vq->num; > + > + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, > + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && > +...

Hi: This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing

...vq size without holding vq mutex. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing metadata prefetch - fix build warning on MIPS Jason Wang (5): vhost: generalize adding used elem vhost: fine grain userspace memory accessors vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() vhost: introduce helpers to get the size of metadata area vhost: access vq metadata through kernel virtual address drivers/vhost/net.c | 4 +- drivers/vhost/vhost.c | 416 +++++++++++++++++++++++++++++++++++++----- drivers/vhost/vhost.h | 15 +- 3 files changed, 3...

...vq size without holding vq mutex. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing metadata prefetch - fix build warning on MIPS Jason Wang (5): vhost: generalize adding used elem vhost: fine grain userspace memory accessors vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() vhost: introduce helpers to get the size of metadata area vhost: access vq metadata through kernel virtual address drivers/vhost/net.c | 4 +- drivers/vhost/vhost.c | 416 +++++++++++++++++++++++++++++++++++++----- drivers/vhost/vhost.h | 15 +- 3 files changed, 3...

...; + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); > } > > static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, > @@ -1193,14 +1198,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, > return true; > } > > -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) > +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) > +{ > + int num = vq->num; > + > + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, > + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && > +...

...; + access_ok(VERIFY_WRITE, device_event, sizeof(*device_event)); > } > > static int vq_access_ok_split(struct vhost_virtqueue *vq, unsigned int num, > @@ -1193,14 +1198,27 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, > return true; > } > > -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) > +int vq_iotlb_prefetch_packed(struct vhost_virtqueue *vq) > +{ > + int num = vq->num; > + > + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, > + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && > +...

...vq size without holding vq mutex. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing metadata prefetch - fix build warning on MIPS Jason Wang (6): vhost: generalize adding used elem vhost: fine grain userspace memory accessors vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() vhost: introduce helpers to get the size of metadata area vhost: factor out setting vring addr and num vhost: access vq metadata through kernel virtual address drivers/vhost/net.c | 4 +- drivers/vhost/vhost.c | 850 ++++++++++++++++++++++++++++++++++++------ dri...

...vq size without holding vq mutex. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing metadata prefetch - fix build warning on MIPS Jason Wang (6): vhost: generalize adding used elem vhost: fine grain userspace memory accessors vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() vhost: introduce helpers to get the size of metadata area vhost: factor out setting vring addr and num vhost: access vq metadata through kernel virtual address drivers/vhost/net.c | 4 +- drivers/vhost/vhost.c | 850 ++++++++++++++++++++++++++++++++++++------ dri...

...size_t hdr_size; > + struct socket *sock; > + struct vhost_net_ubuf_ref *uninitialized_var(ubufs); > + int sent_pkts = 0; why do we need so many locals? > + > + mutex_lock(&vq->mutex); > + sock = vq->private_data; > + if (!sock) > + goto out; > + > + if (!vq_iotlb_prefetch(vq)) > + goto out; > + > + vhost_disable_notify(&net->dev, vq); > + vhost_net_disable_vq(net, vq); > + > + hdr_size = nvq->vhost_hlen; > + > + for (;;) { > + head = vhost_net_tx_get_vq_desc(net, vq, vq->iov, > + ARRAY_SIZE(vq->iov), > + &...

...vq size without holding vq mutex. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing metadata prefetch - fix build warning on MIPS Jason Wang (6): vhost: generalize adding used elem vhost: fine grain userspace memory accessors vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() vhost: introduce helpers to get the size of metadata area vhost: factor out setting vring addr and num vhost: access vq metadata through kernel virtual address drivers/vhost/net.c | 4 +- drivers/vhost/vhost.c | 852 ++++++++++++++++++++++++++++++++++++------ dri...

...t_enable(); r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), - out_num, in_num, NULL, NULL); + out_num, in_num, NULL, NULL, + VHOST_ACCESS_RO); } return r; @@ -377,6 +380,9 @@ static void handle_tx(struct vhost_net *net) if (!sock) goto out; + if (!vq_iotlb_prefetch(vq)) + goto out; + vhost_disable_notify(&net->dev, vq); hdr_size = nvq->vhost_hlen; @@ -564,7 +570,7 @@ static int get_rx_bufs(struct vhost_virtqueue *vq, } r = vhost_get_vq_desc(vq, vq->iov + seg, ARRAY_SIZE(vq->iov) - seg, &out, - &in, log,...

...ree, @@ -1124,6 +1202,10 @@ static int iotlb_access_ok(struct vhost_virtqueue *vq, } size = node->size - addr + node->start; + + if (orig_addr == addr && size >= len) + vhost_vq_meta_update(vq, node, type); + s += size; addr += size; } @@ -1140,13 +1222,15 @@ int vq_iotlb_prefetch(struct vhost_virtqueue *vq) return 1; return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, - num * sizeof *vq->desc) && + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq-...

...ree, @@ -1124,6 +1202,10 @@ static int iotlb_access_ok(struct vhost_virtqueue *vq, } size = node->size - addr + node->start; + + if (orig_addr == addr && size >= len) + vhost_vq_meta_update(vq, node, type); + s += size; addr += size; } @@ -1140,13 +1222,15 @@ int vq_iotlb_prefetch(struct vhost_virtqueue *vq) return 1; return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, - num * sizeof *vq->desc) && + num * sizeof(*vq->desc), VHOST_ADDR_DESC) && iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq-...

...nges from V1: > - instead of pinning pages, use MMU notifier to invalidate vmaps and > remap duing metadata prefetch > - fix build warning on MIPS > > Jason Wang (5): > vhost: generalize adding used elem > vhost: fine grain userspace memory accessors > vhost: rename vq_iotlb_prefetch() to vq_meta_prefetch() > vhost: introduce helpers to get the size of metadata area > vhost: access vq metadata through kernel virtual address > > drivers/vhost/net.c | 4 +- > drivers/vhost/vhost.c | 416 +++++++++++++++++++++++++++++++++++++----- > drivers/vhost/vhost....