Virtualization - Mar 2021 - [RFC v2 7/7] vhost: Route host->guest notification through shadow virtqueue

On 2021/2/9 11:37 ??, Eugenio P?rez wrote:
> Signed-off-by: Eugenio P?rez <eperezma at redhat.com>
> ---
>   hw/virtio/vhost-shadow-virtqueue.h |  2 ++
>   hw/virtio/vhost-shadow-virtqueue.c | 49 ++++++++++++++++++++++++++++++
>   hw/virtio/vhost.c                  |  5 ++-
>   3 files changed, 55 insertions(+), 1 deletion(-)
>
> diff --git a/hw/virtio/vhost-shadow-virtqueue.h
b/hw/virtio/vhost-shadow-virtqueue.h
> index c45035c4b7..210133434c 100644
> --- a/hw/virtio/vhost-shadow-virtqueue.h
> +++ b/hw/virtio/vhost-shadow-virtqueue.h
> @@ -17,6 +17,8 @@
>   
>   typedef struct VhostShadowVirtqueue VhostShadowVirtqueue;
>   
> +EventNotifier *vhost_shadow_vq_get_call_notifier(VhostShadowVirtqueue
*vq);
> +
>   bool vhost_shadow_vq_start_rcu(struct vhost_dev *dev,
>                                  unsigned idx,
>                                  VhostShadowVirtqueue *svq);
> diff --git a/hw/virtio/vhost-shadow-virtqueue.c
b/hw/virtio/vhost-shadow-virtqueue.c
> index 01f282d434..61d98ae652 100644
> --- a/hw/virtio/vhost-shadow-virtqueue.c
> +++ b/hw/virtio/vhost-shadow-virtqueue.c
> @@ -24,6 +24,8 @@ typedef struct VhostShadowVirtqueue {
>   
>       /* Borrowed virtqueue's guest to host notifier. */
>       EventNotifier host_notifier;
> +    /* Host to guest notifier */
> +    EventNotifier *guest_notifier;
>   
>       /* Virtio queue shadowing */
>       VirtQueue *vq;
> @@ -40,6 +42,26 @@ static void vhost_handle_guest_kick(EventNotifier *n)
>       }
>   }
>   
> +/* Forward vhost notifications */
> +static void vhost_handle_call(EventNotifier *n)
> +{
> +    VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
> +                                             call_notifier);
> +
> +    if (event_notifier_test_and_clear(n)) {
> +        event_notifier_set(svq->guest_notifier);
> +    }
> +}

So I wonder how this is synchonized with virtqueue mask/unmask. Or the 
masking is totally transparent to shadow virtqueue?

Thanks

> +
> +/*
> + * Get the vhost call notifier of the shadow vq
> + * @vq Shadow virtqueue
> + */
> +EventNotifier *vhost_shadow_vq_get_call_notifier(VhostShadowVirtqueue *vq)
> +{
> +    return &vq->call_notifier;
> +}
> +
>   /*
>    * Start shadow virtqueue operation.
>    * @dev vhost device
> @@ -57,6 +79,10 @@ bool vhost_shadow_vq_start_rcu(struct vhost_dev *dev,
>           .index = idx,
>           .fd = event_notifier_get_fd(&svq->kick_notifier),
>       };
> +    struct vhost_vring_file call_file = {
> +        .index = idx,
> +        .fd = event_notifier_get_fd(&svq->call_notifier),
> +    };
>       int r;
>   
>       /* Check that notifications are still going directly to vhost dev */
> @@ -66,6 +92,7 @@ bool vhost_shadow_vq_start_rcu(struct vhost_dev *dev,
>                              event_notifier_get_fd(vq_host_notifier));
>       event_notifier_set_handler(&svq->host_notifier,
vhost_handle_guest_kick);
>   
> +    svq->guest_notifier = virtio_queue_get_guest_notifier(svq->vq);
>       r = dev->vhost_ops->vhost_set_vring_kick(dev, &kick_file);
>       if (unlikely(r != 0)) {
>           error_report("Couldn't set kick fd: %s",
strerror(errno));
> @@ -75,8 +102,19 @@ bool vhost_shadow_vq_start_rcu(struct vhost_dev *dev,
>       /* Check for pending notifications from the guest */
>       vhost_handle_guest_kick(&svq->host_notifier);
>   
> +    r = dev->vhost_ops->vhost_set_vring_call(dev, &call_file);
> +    if (r != 0) {
> +        error_report("Couldn't set call fd: %s",
strerror(errno));
> +        goto err_set_vring_call;
> +    }
> +
>       return true;
>   
> +err_set_vring_call:
> +    kick_file.fd = event_notifier_get_fd(vq_host_notifier);
> +    r = dev->vhost_ops->vhost_set_vring_kick(dev, &kick_file);
> +    assert(r == 0);
> +
>   err_set_vring_kick:
>       event_notifier_set_handler(&svq->host_notifier, NULL);
>   
> @@ -108,6 +146,16 @@ void vhost_shadow_vq_stop_rcu(struct vhost_dev *dev,
>       assert(r == 0);
>   
>       event_notifier_set_handler(&svq->host_notifier, NULL);
> +
> +    if (!dev->vqs[idx].notifier_is_masked) {
> +        EventNotifier *e = vhost_shadow_vq_get_call_notifier(svq);
> +
> +        /* Restore vhost call */
> +        vhost_virtqueue_mask(dev, dev->vdev, dev->vq_index + idx,
false);
> +
> +        /* Check for pending calls */
> +        vhost_handle_call(e);
> +    }
>   }
>   
>   /*
> @@ -136,6 +184,7 @@ VhostShadowVirtqueue *vhost_shadow_vq_new(struct
vhost_dev *dev, int idx)
>           goto err_init_call_notifier;
>       }
>   
> +    event_notifier_set_handler(&svq->call_notifier,
vhost_handle_call);
>       return g_steal_pointer(&svq);
>   
>   err_init_call_notifier:
> diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
> index 9d4728e545..0dc95679e9 100644
> --- a/hw/virtio/vhost.c
> +++ b/hw/virtio/vhost.c
> @@ -975,7 +975,6 @@ static int vhost_sw_live_migration_start(struct
vhost_dev *dev)
>           for (idx = 0; idx < dev->nvqs; ++idx) {
>               bool ok = vhost_shadow_vq_start_rcu(dev, idx,
>                                                   dev->shadow_vqs[idx]);
> -
>               if (!ok) {
>                   int stop_idx = idx;
>   
> @@ -1608,6 +1607,10 @@ void vhost_virtqueue_mask(struct vhost_dev *hdev,
VirtIODevice *vdev, int n,
>       if (mask) {
>           assert(vdev->use_guest_notifier_mask);
>           file.fd =
event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
> +    } else if (hdev->sw_lm_enabled) {
> +        VhostShadowVirtqueue *svq = hdev->shadow_vqs[n];
> +        EventNotifier *e = vhost_shadow_vq_get_call_notifier(svq);
> +        file.fd = event_notifier_get_fd(e);
>       } else {
>           file.fd =
event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
>       }