Linux Virtualization - Aug 2019 - [PATCH v3 2/2] virtio: decrement avail idx with buffer detach for packed ring

This patch series fixes the issue with unplug/replug of a port in virtio
console driver which fails with an error "Error allocating inbufs\n".
Patch 1 makes use of 'virtqueue_detach_unused_buf' function to detach
the unused buffers during port hotunplug time.
Patch 2 updates the next avail index for packed ring code.
Tested the packed ring code with the qemu virtio 1.1 device code posted
here [1].

Changes from v2
 Better change log in patch2 - [Greg]
Changes from v1[2]
 Make virtio packed ring code compatible with split ring - [Michael]

[1]  https://marc.info/?l=qemu-devel&m=156471883703948&w=2
[2]  https://lkml.org/lkml/2019/3/4/517

Pankaj Gupta (2):
  virtio_console: free unused buffers with port delete
  virtio: decrement avail idx with buffer detach for packed ring

 char/virtio_console.c |   14 +++++++++++---
 virtio/virtio_ring.c  |    6 ++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
-- 
2.21.0

The commit a7a69ec0d8e4 ("virtio_console: free buffers after reset")
deferred detaching of unused buffer to virtio device unplug time.
This causes unplug/replug of single port in virtio device with an
error "Error allocating inbufs\n". As we don't free the unused
buffers
attached with the port. Re-plug the same port tries to allocate new
buffers in virtqueue and results in this error if queue is full.
This patch removes the unused buffers in vq's when we unplug the port.
This is the best we can do as we cannot call device_reset because virtio
device is still active.

Reported-by: Xiaohui Li <xiaohli at redhat.com>
Fixes: a7a69ec0d8e4 ("virtio_console: free buffers after reset")
Cc: stable at vger.kernel.org
Signed-off-by: Pankaj Gupta <pagupta at redhat.com>
---
 drivers/char/virtio_console.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index 7270e7b69262..e8be82f1bae9 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -1494,15 +1494,25 @@ static void remove_port(struct kref *kref)
 	kfree(port);
 }
 
+static void remove_unused_bufs(struct virtqueue *vq)
+{
+	struct port_buffer *buf;
+
+	while ((buf = virtqueue_detach_unused_buf(vq)))
+		free_buf(buf, true);
+}
+
 static void remove_port_data(struct port *port)
 {
 	spin_lock_irq(&port->inbuf_lock);
 	/* Remove unused data this port might have received. */
 	discard_port_data(port);
+	remove_unused_bufs(port->in_vq);
 	spin_unlock_irq(&port->inbuf_lock);
 
 	spin_lock_irq(&port->outvq_lock);
 	reclaim_consumed_buffers(port);
+	remove_unused_bufs(port->out_vq);
 	spin_unlock_irq(&port->outvq_lock);
 }
 
@@ -1938,11 +1948,9 @@ static void remove_vqs(struct ports_device *portdev)
 	struct virtqueue *vq;
 
 	virtio_device_for_each_vq(portdev->vdev, vq) {
-		struct port_buffer *buf;
 
 		flush_bufs(vq, true);
-		while ((buf = virtqueue_detach_unused_buf(vq)))
-			free_buf(buf, true);
+		remove_unused_bufs(vq);
 	}
 	portdev->vdev->config->del_vqs(portdev->vdev);
 	kfree(portdev->in_vqs);
-- 
2.21.0

This patch decrements 'next_avail_idx' count when detaching a buffer
from vq for packed ring code. Split ring code already does this in
virtqueue_detach_unused_buf_split function. This updates the
'next_avail_idx' to the previous correct index after an unused buffer
is detatched from the vq.

Signed-off-by: Pankaj Gupta <pagupta at redhat.com>
---
 drivers/virtio/virtio_ring.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c8be1c4f5b55..7c69181113e2 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -1537,6 +1537,12 @@ static void *virtqueue_detach_unused_buf_packed(struct
virtqueue *_vq)
 		/* detach_buf clears data, so grab it now. */
 		buf = vq->packed.desc_state[i].data;
 		detach_buf_packed(vq, i, NULL);
+		vq->packed.next_avail_idx--;
+		if (vq->packed.next_avail_idx < 0) {
+			vq->packed.next_avail_idx = vq->packed.vring.num - 1;
+			vq->packed.avail_wrap_counter ^= 1;
+		}
+
 		END_USE(vq);
 		return buf;
 	}
-- 
2.20.1

On Fri, Aug 09, 2019 at 12:18:47PM +0530, Pankaj Gupta
wrote:
> This patch decrements 'next_avail_idx' count when detaching a
buffer
> from vq for packed ring code. Split ring code already does this in
> virtqueue_detach_unused_buf_split function. This updates the
> 'next_avail_idx' to the previous correct index after an unused
buffer
> is detatched from the vq.
> 
> Signed-off-by: Pankaj Gupta <pagupta at redhat.com>
I would make this patch 1, not patch 2, otherwise
patch 1 corrupts the ring.

> ---
>  drivers/virtio/virtio_ring.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index c8be1c4f5b55..7c69181113e2 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1537,6 +1537,12 @@ static void
*virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
>  		/* detach_buf clears data, so grab it now. */
>  		buf = vq->packed.desc_state[i].data;
>  		detach_buf_packed(vq, i, NULL);
> +		vq->packed.next_avail_idx--;
> +		if (vq->packed.next_avail_idx < 0) {
> +			vq->packed.next_avail_idx = vq->packed.vring.num - 1;
> +			vq->packed.avail_wrap_counter ^= 1;
> +		}
> +
>  		END_USE(vq);
>  		return buf;
>  	}
> -- 
> 2.20.1

On Fri, Aug 09, 2019 at 12:18:46PM +0530, Pankaj Gupta
wrote:
> The commit a7a69ec0d8e4 ("virtio_console: free buffers after
reset")
> deferred detaching of unused buffer to virtio device unplug time.
> This causes unplug/replug of single port in virtio device with an
> error "Error allocating inbufs\n". As we don't free the
unused buffers
> attached with the port. Re-plug the same port tries to allocate new
> buffers in virtqueue and results in this error if queue is full.
So why not reuse the buffers that are already there in this case?
Seems quite possible.
> This patch removes the unused buffers in vq's when we unplug the port.
> This is the best we can do as we cannot call device_reset because virtio
> device is still active.
> 
> Reported-by: Xiaohui Li <xiaohli at redhat.com>
> Fixes: a7a69ec0d8e4 ("virtio_console: free buffers after reset")
> Cc: stable at vger.kernel.org
> Signed-off-by: Pankaj Gupta <pagupta at redhat.com>
This is really a revert of a7a69ec0d8e4, just tagged confusingly.

And the original is also supposed to be a bugfix.
So how will the original bug be fixed?

"this is the best we can do" is rarely the case.

I am not necessarily against the revert. But if we go that way then what
we need to do is specify the behaviour in the spec, since strict spec
compliance is exactly what the original patch was addressing.

In particular, we'd document that console has a special property that
when port is detached virtqueue is considered stopped, device must not
use any buffers, and it is legal to take buffers out of the device.


> ---
>  drivers/char/virtio_console.c | 14 +++++++++++---
>  1 file changed, 11 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
> index 7270e7b69262..e8be82f1bae9 100644
> --- a/drivers/char/virtio_console.c
> +++ b/drivers/char/virtio_console.c
> @@ -1494,15 +1494,25 @@ static void remove_port(struct kref *kref)
>  	kfree(port);
>  }
>  
> +static void remove_unused_bufs(struct virtqueue *vq)
> +{
> +	struct port_buffer *buf;
> +
> +	while ((buf = virtqueue_detach_unused_buf(vq)))
> +		free_buf(buf, true);
> +}
> +
>  static void remove_port_data(struct port *port)
>  {
>  	spin_lock_irq(&port->inbuf_lock);
>  	/* Remove unused data this port might have received. */
>  	discard_port_data(port);
> +	remove_unused_bufs(port->in_vq);
>  	spin_unlock_irq(&port->inbuf_lock);
>  
>  	spin_lock_irq(&port->outvq_lock);
>  	reclaim_consumed_buffers(port);
> +	remove_unused_bufs(port->out_vq);
>  	spin_unlock_irq(&port->outvq_lock);
>  }
>  
> @@ -1938,11 +1948,9 @@ static void remove_vqs(struct ports_device *portdev)
>  	struct virtqueue *vq;
>  
>  	virtio_device_for_each_vq(portdev->vdev, vq) {
> -		struct port_buffer *buf;
>  
>  		flush_bufs(vq, true);
> -		while ((buf = virtqueue_detach_unused_buf(vq)))
> -			free_buf(buf, true);
> +		remove_unused_bufs(vq);
>  	}
>  	portdev->vdev->config->del_vqs(portdev->vdev);
>  	kfree(portdev->in_vqs);
> -- 
> 2.21.0

On 2019/8/9 ??2:48, Pankaj Gupta wrote:
> This patch decrements 'next_avail_idx' count when detaching a
buffer
> from vq for packed ring code. Split ring code already does this in
> virtqueue_detach_unused_buf_split function. This updates the
> 'next_avail_idx' to the previous correct index after an unused
buffer
> is detatched from the vq.
>
> Signed-off-by: Pankaj Gupta <pagupta at redhat.com>
> ---
>   drivers/virtio/virtio_ring.c | 6 ++++++
>   1 file changed, 6 insertions(+)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index c8be1c4f5b55..7c69181113e2 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1537,6 +1537,12 @@ static void
*virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
>   		/* detach_buf clears data, so grab it now. */
>   		buf = vq->packed.desc_state[i].data;
>   		detach_buf_packed(vq, i, NULL);
> +		vq->packed.next_avail_idx--;
> +		if (vq->packed.next_avail_idx < 0) {
> +			vq->packed.next_avail_idx = vq->packed.vring.num - 1;
> +			vq->packed.avail_wrap_counter ^= 1;
> +		}
> +
>   		END_USE(vq);
>   		return buf;
>   	}

Acked-by: Jason Wang <jasowang at redhat.com>