search for: vhost_get_avail_idx

...> + &vq->used->idx); > +} > + > #define vhost_get_user(vq, x, ptr, type) \ > ({ \ > int ret; \ > @@ -907,6 +935,43 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) > mutex_unlock(&d->vqs[i]->mutex); > } > > +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, > + __virtio16 *idx) > +{ > + return vhost_get_avail(vq, *idx, &vq->avail->idx); > +} > + > +static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, > + __virtio16 *head, int idx) > +{ > + return vhost_ge...

...> + &vq->used->idx); > +} > + > #define vhost_get_user(vq, x, ptr, type) \ > ({ \ > int ret; \ > @@ -907,6 +935,43 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) > mutex_unlock(&d->vqs[i]->mutex); > } > > +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, > + __virtio16 *idx) > +{ > + return vhost_get_avail(vq, *idx, &vq->avail->idx); > +} > + > +static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, > + __virtio16 *head, int idx) > +{ > + return vhost_ge...

...er(vq, cpu_to_vhost16(vq, vq->last_used_idx), + &vq->used->idx); +} + #define vhost_get_user(vq, x, ptr, type) \ ({ \ int ret; \ @@ -907,6 +935,43 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) mutex_unlock(&d->vqs[i]->mutex); } +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, + __virtio16 *idx) +{ + return vhost_get_avail(vq, *idx, &vq->avail->idx); +} + +static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, + __virtio16 *head, int idx) +{ + return vhost_get_avail(vq, *head, + &vq->ava...

...); >> +} >> + >> #define vhost_get_user(vq, x, ptr, type) \ >> ({ \ >> int ret; \ >> @@ -907,6 +935,43 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) >> mutex_unlock(&d->vqs[i]->mutex); >> } >> >> +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, >> + __virtio16 *idx) >> +{ >> + return vhost_get_avail(vq, *idx, &vq->avail->idx); >> +} >> + >> +static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, >> + __virtio16 *head, int idx) &g...

...eue *vq) static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) { -#if VHOST_ARCH_CAN_ACCEL_UACCESS +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS struct vhost_map *map; struct vring_used *used; @@ -1342,7 +1342,7 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, __virtio16 *idx) { -#if VHOST_ARCH_CAN_ACCEL_UACCESS +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS struct vhost_map *map; struct vring_avail *avail; @@ -1367,7 +1367,7 @@ static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, static inline int vhost_...

...eue *vq) static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) { -#if VHOST_ARCH_CAN_ACCEL_UACCESS +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS struct vhost_map *map; struct vring_used *used; @@ -1342,7 +1342,7 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, __virtio16 *idx) { -#if VHOST_ARCH_CAN_ACCEL_UACCESS +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS struct vhost_map *map; struct vring_avail *avail; @@ -1367,7 +1367,7 @@ static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, static inline int vhost_...

Hi: This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Changes from V1: - instead of pinning pages, use MMU notifier to invalidate vmaps and remap duing

This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. This is done through setup kernel address through vmap() and resigter MMU notifier for invalidation. Test shows about 24% improvement on TX PPS. TCP_STREAM doesn't see obvious improvement.

This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. This is done through setup kernel address through vmap() and resigter MMU notifier for invalidation. Test shows about 24% improvement on TX PPS. TCP_STREAM doesn't see obvious improvement.

...x(struct vhost_virtqueue *vq) > > { > -#if VHOST_ARCH_CAN_ACCEL_UACCESS > +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS > struct vhost_map *map; > struct vring_used *used; > > @@ -1342,7 +1342,7 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) > static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, > __virtio16 *idx) > { > -#if VHOST_ARCH_CAN_ACCEL_UACCESS > +#ifdef VHOST_ARCH_CAN_ACCEL_UACCESS > struct vhost_map *map; > struct vring_avail *avail; > > @@ -1367,7 +1367,7 @@ static inline int vhost_get_avail_idx(struct vhost_...

Hi: This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Please review Jason Wang (3): vhost: generalize adding used elem vhost: fine grain userspace memory

Hi: This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Please review Jason Wang (3): vhost: generalize adding used elem vhost: fine grain userspace memory

...x; > > - if (vq->avail_idx == vq->last_avail_idx) { > + if (unlikely(vq->avail_idx == vq->last_avail_idx)) { > + /* If we already have work to do, don't bother re-checking. */ > + if (likely(vq->ndescs)) > + return vq->num; > + > if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { > vq_err(vq, "Failed to access avail idx at %p\n", > &vq->avail->idx); > @@ -2541,6 +2549,24 @@ static int fetch_descs(struct vhost_virtqueue *vq) > return 0; > } > > +static int fetch_descs(struct vhost_virtqueue *...

...x; > > - if (vq->avail_idx == vq->last_avail_idx) { > + if (unlikely(vq->avail_idx == vq->last_avail_idx)) { > + /* If we already have work to do, don't bother re-checking. */ > + if (likely(vq->ndescs)) > + return vq->num; > + > if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { > vq_err(vq, "Failed to access avail idx at %p\n", > &vq->avail->idx); > @@ -2541,6 +2549,24 @@ static int fetch_descs(struct vhost_virtqueue *vq) > return 0; > } > > +static int fetch_descs(struct vhost_virtqueue *...

...l_idx; >>> + int ret; >>> + >>> + /* Check it isn't doing very strange things with descriptor numbers. */ >>> + last_avail_idx = vq->last_avail_idx; >>> + >>> + if (vq->avail_idx == vq->last_avail_idx) { >>> + if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { >>> + vq_err(vq, "Failed to access avail idx at %p\n", >>> + &vq->avail->idx); >>> + return -EFAULT; >>> + } >>> + vq->avail_idx = vhost16_to_cpu(vq, avail_idx); >>> + >>> + if (u...

This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Changes from V2: - fix buggy range overlapping check - tear down MMU notifier during vhost ioctl to make sure

This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. Test shows about 24% improvement on TX PPS. It should benefit other cases as well. Changes from V2: - fix buggy range overlapping check - tear down MMU notifier during vhost ioctl to make sure

This series tries to access virtqueue metadata through kernel virtual address instead of copy_user() friends since they had too much overheads like checks, spec barriers or even hardware feature toggling. This is done through setup kernel address through direct mapping and co-opreate VM management with MMU notifiers. Test shows about 23% improvement on TX PPS. TCP_STREAM doesn't see obvious

...; - __virtio16 avail_idx; > - __virtio16 ring_head; > - int ret, access; > - > - /* Check it isn't doing very strange things with descriptor numbers. */ > - last_avail_idx = vq->last_avail_idx; > - > - if (vq->avail_idx == vq->last_avail_idx) { > - if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { > - vq_err(vq, "Failed to access avail idx at %p\n", > - &vq->avail->idx); > - return -EFAULT; > - } > - vq->avail_idx = vhost16_to_cpu(vq, avail_idx); > - > - if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq...

...; - __virtio16 avail_idx; > - __virtio16 ring_head; > - int ret, access; > - > - /* Check it isn't doing very strange things with descriptor numbers. */ > - last_avail_idx = vq->last_avail_idx; > - > - if (vq->avail_idx == vq->last_avail_idx) { > - if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { > - vq_err(vq, "Failed to access avail idx at %p\n", > - &vq->avail->idx); > - return -EFAULT; > - } > - vq->avail_idx = vhost16_to_cpu(vq, avail_idx); > - > - if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq...