Virtualization - Nov 2014 - [PATCH 1/2] virito: introduce methods of fixing device features

Buggy host may advertised buggy host features (a usual case is that host
advertise a feature whose dependencies were missed). In this case, driver
should detect and disable the buggy features by itself.

This patch introduces driver specific fix_features() method which is called
just before features finalizing to detect and disable buggy features
advertised by host.

Virtio-net will be the first user.

Cc: Rusty Russell <rusty at rustcorp.com.au>
Cc: Michael S. Tsirkin <mst at redhat.com>
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
 drivers/virtio/virtio.c       |  4 ++++
 include/linux/virtio.h        |  1 +
 include/linux/virtio_config.h | 12 ++++++++++++
 3 files changed, 17 insertions(+)

diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index df598dd..7001d6e 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -181,6 +181,10 @@ static int virtio_dev_probe(struct device *_d)
 		if (device_features & (1 << i))
 			set_bit(i, dev->features);
 
+	/* Fix buggy features advertised by host */
+	if (drv->fix_features)
+		drv->fix_features(dev);
+
 	dev->config->finalize_features(dev);
 
 	err = drv->probe(dev);
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 65261a7..9d01b54 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -142,6 +142,7 @@ struct virtio_driver {
 	void (*scan)(struct virtio_device *dev);
 	void (*remove)(struct virtio_device *dev);
 	void (*config_changed)(struct virtio_device *dev);
+	void (*fix_features)(struct virtio_device *dev);
 #ifdef CONFIG_PM
 	int (*freeze)(struct virtio_device *dev);
 	int (*restore)(struct virtio_device *dev);
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 7f4ef66..7bd89ea 100644
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -96,6 +96,18 @@ static inline bool virtio_has_feature(const struct
virtio_device *vdev,
 	return test_bit(fbit, vdev->features);
 }
 
+static inline void virtio_disable_feature(struct virtio_device *vdev,
+                                          unsigned int fbit)
+{
+	BUG_ON(fbit >= VIRTIO_TRANSPORT_F_START);
+	BUG_ON(vdev->config->get_status(vdev) &
+	       ~(VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER));
+
+	virtio_check_driver_offered_feature(vdev, fbit);
+
+	clear_bit(fbit, vdev->features);
+}
+
 static inline
 struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
 					vq_callback_t *c, const char *n)
-- 
1.9.1

This patch tries to detect the possible buggy features advertised by host
and fix them. One example is booting virtio-net with only ctrl_vq disabled,
qemu may still advertise many features which depends on it. This will
trigger several BUG()s in virtnet_send_command().

This patch utilizes the fix_features() method, and disables all features that
depends on ctrl_vq if it was not advertised.

This fixes the crash when booting with ctrl_vq=off.

Cc: Rusty Russell <rusty at rustcorp.com.au>
Cc: Michael S. Tsirkin <mst at redhat.com>
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
Changes from V1:
- fix the cut-and-paste error
---
 drivers/net/virtio_net.c | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index ec2a8b4..6ce125e 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -1948,6 +1948,40 @@ static int virtnet_restore(struct virtio_device *vdev)
 }
 #endif
 
+static void virtnet_fix_features(struct virtio_device *dev)
+{
+	if (!virtio_has_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
+		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_RX)) {
+			pr_warning("Disable VIRTIO_NET_F_CTRL_RX since host "
+				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
+			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_RX);
+		}
+		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) {
+			pr_warning("Disable VIRTIO_NET_F_CTRL_VLAN since host "
+				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
+			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_VLAN);
+		}
+		if (virtio_has_feature(dev, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
+			pr_warning("Disable VIRTIO_NET_F_GUEST_ANNOUNCE since "
+				   "host does not advertise "
+				   "VIRTIO_NET_F_CTRL_VQ");
+			virtio_disable_feature(dev,
+					       VIRTIO_NET_F_GUEST_ANNOUNCE);
+		}
+		if (virtio_has_feature(dev, VIRTIO_NET_F_MQ)) {
+			pr_warning("Disable VIRTIO_NET_F_MQ since host "
+				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
+			virtio_disable_feature(dev, VIRTIO_NET_F_MQ);
+		}
+		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
+			pr_warning("Disable VIRTIO_NET_F_CTRL_MAC_ADDR since "
+				   "host does not advertise "
+				   "VIRTIO_NET_F_CTRL_VQ");
+			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR);
+		}
+	}
+}
+
 static struct virtio_device_id id_table[] = {
 	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
 	{ 0 },
@@ -1975,6 +2009,7 @@ static struct virtio_driver virtio_net_driver = {
 	.probe =	virtnet_probe,
 	.remove =	virtnet_remove,
 	.config_changed = virtnet_config_changed,
+	.fix_features = virtnet_fix_features,
 #ifdef CONFIG_PM_SLEEP
 	.freeze =	virtnet_freeze,
 	.restore =	virtnet_restore,
-- 
1.9.1

On 11/13/2014 01:52 PM, Jason Wang wrote:
> This patch tries to detect the possible buggy features advertised by host
> and fix them. One example is booting virtio-net with only ctrl_vq disabled,
> qemu may still advertise many features which depends on it. This will
> trigger several BUG()s in virtnet_send_command().
> 
> This patch utilizes the fix_features() method, and disables all features
that
> depends on ctrl_vq if it was not advertised.
> 
> This fixes the crash when booting with ctrl_vq=off.
> 
> Cc: Rusty Russell <rusty at rustcorp.com.au>
> Cc: Michael S. Tsirkin <mst at redhat.com>
> Signed-off-by: Jason Wang <jasowang at redhat.com>
> ---
> Changes from V1:
> - fix the cut-and-paste error
> ---
>  drivers/net/virtio_net.c | 35 +++++++++++++++++++++++++++++++++++
>  1 file changed, 35 insertions(+)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index ec2a8b4..6ce125e 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -1948,6 +1948,40 @@ static int virtnet_restore(struct virtio_device
*vdev)
>  }
>  #endif
>  
> +static void virtnet_fix_features(struct virtio_device *dev)
> +{
> +	if (!virtio_has_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_RX)) {
> +			pr_warning("Disable VIRTIO_NET_F_CTRL_RX since host "
> +				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_RX);
> +		}
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) {
> +			pr_warning("Disable VIRTIO_NET_F_CTRL_VLAN since host "
> +				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_VLAN);
> +		}
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
> +			pr_warning("Disable VIRTIO_NET_F_GUEST_ANNOUNCE since "
> +				   "host does not advertise "
> +				   "VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev,
> +					       VIRTIO_NET_F_GUEST_ANNOUNCE);
> +		}
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_MQ)) {
> +			pr_warning("Disable VIRTIO_NET_F_MQ since host "
> +				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev, VIRTIO_NET_F_MQ);
> +		}
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
> +			pr_warning("Disable VIRTIO_NET_F_CTRL_MAC_ADDR since "
> +				   "host does not advertise "
> +				   "VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR);
> +		}

Can we use a feature array and check with one loop?  The current check looks so
dup?


Thanks,
Wanlong Gao
> +	}
> +}
> +
>  static struct virtio_device_id id_table[] = {
>  	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
>  	{ 0 },
> @@ -1975,6 +2009,7 @@ static struct virtio_driver virtio_net_driver = {
>  	.probe =	virtnet_probe,
>  	.remove =	virtnet_remove,
>  	.config_changed = virtnet_config_changed,
> +	.fix_features = virtnet_fix_features,
>  #ifdef CONFIG_PM_SLEEP
>  	.freeze =	virtnet_freeze,
>  	.restore =	virtnet_restore,
>

On Thu, 13 Nov 2014 13:52:53 +0800
Jason Wang <jasowang at redhat.com> wrote:

typo in subject-prefix: s/virito/virtio/
> Buggy host may advertised buggy host features (a usual case is that host
> advertise a feature whose dependencies were missed). In this case, driver
> should detect and disable the buggy features by itself.
> 
> This patch introduces driver specific fix_features() method which is called
> just before features finalizing to detect and disable buggy features
> advertised by host.
So the basic problem this patch fixes is that an individual driver may
only specify a static set of features but cannot specify any
dependencies, right? Adding a sanitizer step makes sense, I guess.
> 
> Virtio-net will be the first user.
> 
> Cc: Rusty Russell <rusty at rustcorp.com.au>
> Cc: Michael S. Tsirkin <mst at redhat.com>
> Signed-off-by: Jason Wang <jasowang at redhat.com>
> ---
>  drivers/virtio/virtio.c       |  4 ++++
>  include/linux/virtio.h        |  1 +
>  include/linux/virtio_config.h | 12 ++++++++++++
>  3 files changed, 17 insertions(+)
> 
> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> index df598dd..7001d6e 100644
> --- a/drivers/virtio/virtio.c
> +++ b/drivers/virtio/virtio.c
> @@ -181,6 +181,10 @@ static int virtio_dev_probe(struct device *_d)
>  		if (device_features & (1 << i))
>  			set_bit(i, dev->features);
> 
> +	/* Fix buggy features advertised by host */
> +	if (drv->fix_features)
> +		drv->fix_features(dev);
I'd probably call this "sanitize_features" instead.
> +
>  	dev->config->finalize_features(dev);
> 
>  	err = drv->probe(dev);
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 7f4ef66..7bd89ea 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -96,6 +96,18 @@ static inline bool virtio_has_feature(const struct
virtio_device *vdev,
>  	return test_bit(fbit, vdev->features);
>  }
> 
> +static inline void virtio_disable_feature(struct virtio_device *vdev,
> +                                          unsigned int fbit)
> +{
> +	BUG_ON(fbit >= VIRTIO_TRANSPORT_F_START);
> +	BUG_ON(vdev->config->get_status(vdev) &
> +	       ~(VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER));
When we add virtio-1 support, we can add a check for FEATURES_OK here,
so we're really on the safe side.
> +
> +	virtio_check_driver_offered_feature(vdev, fbit);
> +
> +	clear_bit(fbit, vdev->features);
> +}
> +
>  static inline
>  struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
>  					vq_callback_t *c, const char *n)
The approach looks good to me.

On Thu, 13 Nov 2014 13:52:54 +0800
Jason Wang <jasowang at redhat.com> wrote:
> This patch tries to detect the possible buggy features advertised by host
> and fix them. One example is booting virtio-net with only ctrl_vq disabled,
> qemu may still advertise many features which depends on it. This will
> trigger several BUG()s in virtnet_send_command().
> 
> This patch utilizes the fix_features() method, and disables all features
that
> depends on ctrl_vq if it was not advertised.
> 
> This fixes the crash when booting with ctrl_vq=off.
That's a qemu device property, right? Might want to mention that, as
this line sounds like it is a kernel parameter.
> 
> Cc: Rusty Russell <rusty at rustcorp.com.au>
> Cc: Michael S. Tsirkin <mst at redhat.com>
> Signed-off-by: Jason Wang <jasowang at redhat.com>
> ---
> Changes from V1:
> - fix the cut-and-paste error
> ---
>  drivers/net/virtio_net.c | 35 +++++++++++++++++++++++++++++++++++
>  1 file changed, 35 insertions(+)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index ec2a8b4..6ce125e 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -1948,6 +1948,40 @@ static int virtnet_restore(struct virtio_device
*vdev)
>  }
>  #endif
> 
> +static void virtnet_fix_features(struct virtio_device *dev)
> +{
> +	if (!virtio_has_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
> +		if (virtio_has_feature(dev, VIRTIO_NET_F_CTRL_RX)) {
> +			pr_warning("Disable VIRTIO_NET_F_CTRL_RX since host "
> +				   "does not advertise VIRTIO_NET_F_CTRL_VQ");
> +			virtio_disable_feature(dev, VIRTIO_NET_F_CTRL_RX);
> +		}
You should probably use dev_warn() or so, so that the user can figure
out which device the message is for. And perhaps add "buggy
hypervisor"
to the message to make clear that it's not a guest problem.

I also like the suggestion to use a dependency array.

On 11/13/2014 04:46 PM, Cornelia Huck wrote:
> On Thu, 13 Nov 2014 13:52:53 +0800
> Jason Wang <jasowang at redhat.com> wrote:
>
> typo in subject-prefix: s/virito/virtio/
>
Will correct this.
>> Buggy host may advertised buggy host features (a usual case is that
host
>> advertise a feature whose dependencies were missed). In this case,
driver
>> should detect and disable the buggy features by itself.
>>
>> This patch introduces driver specific fix_features() method which is
called
>> just before features finalizing to detect and disable buggy features
>> advertised by host.
> So the basic problem this patch fixes is that an individual driver may
> only specify a static set of features but cannot specify any
> dependencies, right?
Right, and what even worse is qemu could not handle dependencies as
well. So we need fix both sides.
>  Adding a sanitizer step makes sense, I guess.
>
>> Virtio-net will be the first user.
>>
>> Cc: Rusty Russell <rusty at rustcorp.com.au>
>> Cc: Michael S. Tsirkin <mst at redhat.com>
>> Signed-off-by: Jason Wang <jasowang at redhat.com>
>> ---
>>  drivers/virtio/virtio.c       |  4 ++++
>>  include/linux/virtio.h        |  1 +
>>  include/linux/virtio_config.h | 12 ++++++++++++
>>  3 files changed, 17 insertions(+)
>>
>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>> index df598dd..7001d6e 100644
>> --- a/drivers/virtio/virtio.c
>> +++ b/drivers/virtio/virtio.c
>> @@ -181,6 +181,10 @@ static int virtio_dev_probe(struct device *_d)
>>  		if (device_features & (1 << i))
>>  			set_bit(i, dev->features);
>>
>> +	/* Fix buggy features advertised by host */
>> +	if (drv->fix_features)
>> +		drv->fix_features(dev);
> I'd probably call this "sanitize_features" instead.
Ok.
>> +
>>  	dev->config->finalize_features(dev);
>>
>>  	err = drv->probe(dev);
>> diff --git a/include/linux/virtio_config.h
b/include/linux/virtio_config.h
>> index 7f4ef66..7bd89ea 100644
>> --- a/include/linux/virtio_config.h
>> +++ b/include/linux/virtio_config.h
>> @@ -96,6 +96,18 @@ static inline bool virtio_has_feature(const struct
virtio_device *vdev,
>>  	return test_bit(fbit, vdev->features);
>>  }
>>
>> +static inline void virtio_disable_feature(struct virtio_device *vdev,
>> +                                          unsigned int fbit)
>> +{
>> +	BUG_ON(fbit >= VIRTIO_TRANSPORT_F_START);
>> +	BUG_ON(vdev->config->get_status(vdev) &
>> +	       ~(VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER));
> When we add virtio-1 support, we can add a check for FEATURES_OK here,
> so we're really on the safe side.
>
If I read the spec correctly, FEATURES_OK was set only after writing the
features bits to device. But we want to sanitize the them
before.
>> +
>> +	virtio_check_driver_offered_feature(vdev, fbit);
>> +
>> +	clear_bit(fbit, vdev->features);
>> +}
>> +
>>  static inline
>>  struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
>>  					vq_callback_t *c, const char *n)
> The approach looks good to me.
>
Thanks for the review.

Jason Wang <jasowang at redhat.com> writes:
> Buggy host may advertised buggy host features (a usual case is that host
> advertise a feature whose dependencies were missed). In this case, driver
> should detect and disable the buggy features by itself.
Sorry, I've been focussing elsewhere.

I would really prefer that drivers offer a "feature_depends" table,
which can indicate that feature A depends on feature B, and have the
core iterate, complain and fixup as necessary.

Is that expressive enough, or do we need more?

Thanks,
Rusty.

----- Original Message -----
> Jason Wang <jasowang at redhat.com> writes:
> > Buggy host may advertised buggy host features (a usual case is that
host
> > advertise a feature whose dependencies were missed). In this case,
driver
> > should detect and disable the buggy features by itself.
> 
> Sorry, I've been focussing elsewhere.
> 
> I would really prefer that drivers offer a "feature_depends"
table,
> which can indicate that feature A depends on feature B, and have the
> core iterate, complain and fixup as necessary.
> 
> Is that expressive enough, or do we need more?
> 
> Thanks,
> Rusty.
I suggested something like this in https://lkml.org/lkml/2014/11/19/132, but
Michael does not like it ...