Linux Virtualization - Jul 2012 - [RFC 0/2] virtio: provide a way for host to monitor critical events in the device

As it was discussed recently, there's currently no way for the guest to
notify
the host about panics. Further more, there's no reasonable way to notify the
host of other critical events such as an OOM kill.

This short patch series introduces a new device named virtio-notifier which
does two simple things:

 1. Provide a simple interface for the guest to notify the host of critical
 events. This is easily expandible to add support for any events we may find
 interesting for the host to know about.

 2. Provide an "echo" interface for the host to ping the guest. This
allows
 the host to ping the guest at intervals chosen by the host, and act
 accordingly if no response has been received.

Sasha Levin (2):
  virtio: Introduce virtio-notifier
  kvm tools: support virtio-notifier

 drivers/virtio/Kconfig                  |   11 ++
 drivers/virtio/Makefile                 |    1 +
 drivers/virtio/virtio_notifier.c        |  135 ++++++++++++++++++++
 include/linux/virtio_ids.h              |    1 +
 include/linux/virtio_notifier.h         |   15 +++
 tools/kvm/Makefile                      |    1 +
 tools/kvm/builtin-run.c                 |    6 +
 tools/kvm/include/kvm/virtio-notifier.h |    9 ++
 tools/kvm/include/kvm/virtio-pci-dev.h  |    1 +
 tools/kvm/virtio/notifier.c             |  203 +++++++++++++++++++++++++++++++
 10 files changed, 383 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virtio/virtio_notifier.c
 create mode 100644 include/linux/virtio_notifier.h
 create mode 100644 tools/kvm/include/kvm/virtio-notifier.h
 create mode 100644 tools/kvm/virtio/notifier.c

-- 
1.7.8.6

[TODO: Find a better name]

virtio-notifier is a new driver which provides guests the ability to report
critical events such as a panic or OOM to the host.

The driver also provides an "echo" channel which is primed with a
buffer when
the driver is initialized, and allows the host to "ping" the guest to
make
sure the guest is still alive and well.

Signed-off-by: Sasha Levin <levinsasha928 at gmail.com>
---
 drivers/virtio/Kconfig           |   11 +++
 drivers/virtio/Makefile          |    1 +
 drivers/virtio/virtio_notifier.c |  135 ++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_ids.h       |    1 +
 include/linux/virtio_notifier.h  |   15 ++++
 5 files changed, 163 insertions(+), 0 deletions(-)
 create mode 100644 drivers/virtio/virtio_notifier.c
 create mode 100644 include/linux/virtio_notifier.h

diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 1a61939..1be8f93 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -46,4 +46,15 @@ config VIRTIO_BALLOON
 
  	 If unsure, say N.
 
+config VIRTIO_NOTIFIER
+	tristate "Virtio notifier driver (EXPERIMENTAL)"
+	select VIRTIO
+	select VIRTIO_RING
+	---help---
+	This driver provides support for passing improtant notifications such as
+	notification about guest PANIC or OOM back to the host.
+
+	Also, the driver provides a mechanism to detect lockups in the guest (similar
+	to a watchdog), notifying the host about such lockups.
+
 endmenu
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index 5a4c63c..7b77d0b 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -3,3 +3,4 @@ obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
 obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
 obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
+obj-$(CONFIG_VIRTIO_NOTIFIER) += virtio_notifier.o
diff --git a/drivers/virtio/virtio_notifier.c b/drivers/virtio/virtio_notifier.c
new file mode 100644
index 0000000..77393c5
--- /dev/null
+++ b/drivers/virtio/virtio_notifier.c
@@ -0,0 +1,135 @@
+/*
+ * Notifier/watchdog driver for virtio
+ *  Copyright (C) 2012 Sasha Levin
+ */
+
+#include <linux/err.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+#include <linux/virtio.h>
+#include <linux/virtio_notifier.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/oom.h>
+
+static struct virtqueue *echo, *notif;
+static u32 notif_cnt[VIRTIO_NOTIF_CNT];
+static u64 echo_val;
+
+static void notif_done(struct virtqueue *vq)
+{
+}
+
+static void echo_done(struct virtqueue *vq)
+{
+	struct scatterlist sg;
+
+        sg_init_one(&sg, &echo_val, sizeof(echo_val));
+	if (virtqueue_add_buf(echo, &sg, 1, 0, echo, GFP_KERNEL) < 0)
+		BUG();
+
+	virtqueue_kick(echo);
+}
+
+static void notify_host(int notification)
+{
+	struct scatterlist sg;
+
+	notif_cnt[notification]++;
+
+        sg_init_one(&sg, notif_cnt, sizeof(notif_cnt));
+
+	if (virtqueue_add_buf(notif, &sg, 0, 1, notif, GFP_KERNEL) < 0)
+		BUG();
+
+	virtqueue_kick(notif);
+}
+
+static int virtio_notif_panic(struct notifier_block *this, unsigned long ev,
void *ptr)
+{
+	notify_host(VIRTIO_NOTIF_PANIC);
+
+	return NOTIFY_DONE;
+}
+
+static int virtio_notif_oom(struct notifier_block *this, unsigned long ev, void
*ptr)
+{
+	notify_host(VIRTIO_NOTIF_OOM);
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block virtio_notif_panic_block = {
+        .notifier_call = virtio_notif_panic,
+};
+
+static struct notifier_block virtio_notif_oom_block = {
+	.notifier_call = virtio_notif_oom,
+};
+
+static int virtnotif_probe(struct virtio_device *vdev)
+{
+	int err;
+	struct virtqueue *vqs[2];
+	vq_callback_t *callbacks[] = { notif_done, echo_done };
+	const char *names[] = { "notif", "echo" };
+	struct scatterlist sg;
+	
+	/* We expect two virtqueue. */
+	err = vdev->config->find_vqs(vdev, 2, vqs, callbacks, names);
+	if (err)
+		return err;
+
+	notif = vqs[0];
+	echo = vqs[1];
+
+	/*
+	 * Prime this virtqueue with one buffer so the hypervisor can
+	 * use it to signal us later.
+	 */
+	sg_init_one(&sg, &echo_val, sizeof(echo_val));
+	if (virtqueue_add_buf(echo, &sg, 1, 0, echo, GFP_KERNEL) < 0)
+		BUG();
+
+	virtqueue_kick(echo);
+
+	atomic_notifier_chain_register(&panic_notifier_list,
&virtio_notif_panic_block);
+	register_oom_notifier(&virtio_notif_oom_block);
+
+	return 0;
+}
+
+static void __devexit virtnotif_remove(struct virtio_device *vdev)
+{
+	vdev->config->reset(vdev);
+	vdev->config->del_vqs(vdev);
+}
+
+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_NOTIFIER, VIRTIO_DEV_ANY_ID },
+	{ 0 },
+};
+
+static struct virtio_driver virtio_notifier_driver = {
+	.driver.name =	KBUILD_MODNAME,
+	.driver.owner =	THIS_MODULE,
+	.id_table =	id_table,
+	.probe =	virtnotif_probe,
+	.remove =	__devexit_p(virtnotif_remove),
+};
+
+static int __init init(void)
+{
+	return register_virtio_driver(&virtio_notifier_driver);
+}
+
+static void __exit fini(void)
+{
+	unregister_virtio_driver(&virtio_notifier_driver);
+}
+module_init(init);
+module_exit(fini);
+
+MODULE_DEVICE_TABLE(virtio, id_table);
+MODULE_DESCRIPTION("Virtio notifier driver");
+MODULE_LICENSE("GPL");
diff --git a/include/linux/virtio_ids.h b/include/linux/virtio_ids.h
index 7529b85..553edf9 100644
--- a/include/linux/virtio_ids.h
+++ b/include/linux/virtio_ids.h
@@ -34,6 +34,7 @@
 #define VIRTIO_ID_CONSOLE	3 /* virtio console */
 #define VIRTIO_ID_RNG		4 /* virtio ring */
 #define VIRTIO_ID_BALLOON	5 /* virtio balloon */
+#define VIRTIO_ID_NOTIFIER      6 /* virtio notifier */
 #define VIRTIO_ID_RPMSG		7 /* virtio remote processor messaging */
 #define VIRTIO_ID_SCSI		8 /* virtio scsi */
 #define VIRTIO_ID_9P		9 /* 9p virtio console */
diff --git a/include/linux/virtio_notifier.h b/include/linux/virtio_notifier.h
new file mode 100644
index 0000000..ad5bcf7
--- /dev/null
+++ b/include/linux/virtio_notifier.h
@@ -0,0 +1,15 @@
+#ifndef _LINUX_VIRTIO_NOTIFIER_H
+#define _LINUX_VIRTIO_NOTIFIER_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers. */
+#include <linux/virtio_ids.h>
+#include <linux/virtio_config.h>
+
+enum virtio_notifications {
+	VIRTIO_NOTIF_PANIC,
+	VIRTIO_NOTIF_OOM,
+
+	VIRTIO_NOTIF_CNT
+};
+
+#endif /* _LINUX_VIRTIO_RNG_H */
-- 
1.7.8.6

This patch supports the new virtio-notifier driver.

When the guest experiences a panic or an OOM, it will notify the host and
the host will notify the user of the event.

We also ping echo packets every second to the guest to see if it's still
alive and well, I haven't actually taken care of the case when echos are
gone for some reason since I'm trying to think of something useful to do
with that besides yell on stdout.

Signed-off-by: Sasha Levin <levinsasha928 at gmail.com>
---
 tools/kvm/Makefile                      |    1 +
 tools/kvm/builtin-run.c                 |    6 +
 tools/kvm/include/kvm/virtio-notifier.h |    9 ++
 tools/kvm/include/kvm/virtio-pci-dev.h  |    1 +
 tools/kvm/virtio/notifier.c             |  203 +++++++++++++++++++++++++++++++
 5 files changed, 220 insertions(+), 0 deletions(-)
 create mode 100644 tools/kvm/include/kvm/virtio-notifier.h
 create mode 100644 tools/kvm/virtio/notifier.c

diff --git a/tools/kvm/Makefile b/tools/kvm/Makefile
index f9e1ec1..6827c1f 100644
--- a/tools/kvm/Makefile
+++ b/tools/kvm/Makefile
@@ -89,6 +89,7 @@ OBJS	+= hw/pci-shmem.o
 OBJS	+= kvm-ipc.o
 OBJS	+= builtin-sandbox.o
 OBJS	+= virtio/mmio.o
+OBJS	+= virtio/notifier.o
 
 # Translate uname -m into ARCH string
 ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/)
diff --git a/tools/kvm/builtin-run.c b/tools/kvm/builtin-run.c
index 750d30c..90ac622 100644
--- a/tools/kvm/builtin-run.c
+++ b/tools/kvm/builtin-run.c
@@ -33,6 +33,7 @@
 #include "kvm/pci-shmem.h"
 #include "kvm/kvm-ipc.h"
 #include "kvm/builtin-debug.h"
+#include "kvm/virtio-notifier.h"
 
 #include <linux/types.h>
 #include <linux/err.h>
@@ -99,6 +100,7 @@ static bool using_rootfs;
 static bool custom_rootfs;
 static bool no_net;
 static bool no_dhcp;
+static bool virtio_notifier;
 extern bool ioport_debug;
 extern bool mmio_debug;
 static int  kvm_run_wrapper;
@@ -442,6 +444,7 @@ static const struct option options[] = {
 	OPT_BOOLEAN('\0', "vnc", &vnc, "Enable VNC
framebuffer"),
 	OPT_BOOLEAN('\0', "sdl", &sdl, "Enable SDL
framebuffer"),
 	OPT_BOOLEAN('\0', "rng", &virtio_rng, "Enable
virtio Random Number Generator"),
+	OPT_BOOLEAN('\0', "notifier", &virtio_notifier,
"Enable virtio notifier"),
 	OPT_CALLBACK('\0', "9p", NULL,
"dir_to_share,tag_name",
 		     "Enable virtio 9p to share files between host and guest",
virtio_9p_rootdir_parser),
 	OPT_STRING('\0', "console", &console, "serial,
virtio or hv",
@@ -1182,6 +1185,9 @@ static int kvm_cmd_run_init(int argc, const char **argv)
 	if (virtio_rng)
 		virtio_rng__init(kvm);
 
+	if (virtio_notifier)
+		virtio_notif__init(kvm);
+
 	if (balloon)
 		virtio_bln__init(kvm);
 
diff --git a/tools/kvm/include/kvm/virtio-notifier.h
b/tools/kvm/include/kvm/virtio-notifier.h
new file mode 100644
index 0000000..5673fc2
--- /dev/null
+++ b/tools/kvm/include/kvm/virtio-notifier.h
@@ -0,0 +1,9 @@
+#ifndef KVM__NOTIF_VIRTIO_H
+#define KVM__NOTIF_VIRTIO_H
+
+struct kvm;
+
+int virtio_notif__init(struct kvm *kvm);
+int virtio_notif__exit(struct kvm *kvm);
+
+#endif /* KVM__RNG_VIRTIO_H */
diff --git a/tools/kvm/include/kvm/virtio-pci-dev.h
b/tools/kvm/include/kvm/virtio-pci-dev.h
index 7ceb125..a387ecd 100644
--- a/tools/kvm/include/kvm/virtio-pci-dev.h
+++ b/tools/kvm/include/kvm/virtio-pci-dev.h
@@ -13,6 +13,7 @@
 #define PCI_DEVICE_ID_VIRTIO_CONSOLE		0x1003
 #define PCI_DEVICE_ID_VIRTIO_RNG		0x1004
 #define PCI_DEVICE_ID_VIRTIO_BLN		0x1005
+#define PCI_DEVICE_ID_VIRTIO_NOTIFIER		0x1006
 #define PCI_DEVICE_ID_VIRTIO_9P			0x1009
 #define PCI_DEVICE_ID_VESA			0x2000
 #define PCI_DEVICE_ID_PCI_SHMEM			0x0001
diff --git a/tools/kvm/virtio/notifier.c b/tools/kvm/virtio/notifier.c
new file mode 100644
index 0000000..c17546b
--- /dev/null
+++ b/tools/kvm/virtio/notifier.c
@@ -0,0 +1,203 @@
+#include "kvm/virtio-notifier.h"
+
+#include "kvm/virtio-pci-dev.h"
+
+#include "kvm/virtio.h"
+#include "kvm/util.h"
+#include "kvm/kvm.h"
+#include "kvm/threadpool.h"
+#include "kvm/guest_compat.h"
+
+#include <linux/virtio_ring.h>
+#include <linux/virtio_notifier.h>
+
+#include <linux/list.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <pthread.h>
+#include <linux/kernel.h>
+
+#define NUM_VIRT_QUEUES			2
+#define VIRTIO_NOTIFIER_QUEUE_SIZE	128
+
+struct ntf_dev_job {
+	struct virt_queue	*vq;
+	struct ntf_dev		*ndev;
+	struct thread_pool__job	job_id;
+};
+
+struct ntf_dev {
+	struct list_head	list;
+	struct virtio_device	vdev;
+
+	/* virtio queue */
+	struct virt_queue	vqs[NUM_VIRT_QUEUES];
+	struct ntf_dev_job	jobs[NUM_VIRT_QUEUES];
+};
+
+static LIST_HEAD(ndevs);
+static int compat_id = -1;
+
+static void set_config(struct kvm *kvm, void *dev, u8 data, u32 offset)
+{
+	/* Unused */
+}
+
+static u8 get_config(struct kvm *kvm, void *dev, u32 offset)
+{
+	/* Unused */
+	return 0;
+}
+
+static u32 get_host_features(struct kvm *kvm, void *dev)
+{
+	/* Unused */
+	return 0;
+}
+
+static void set_guest_features(struct kvm *kvm, void *dev, u32 features)
+{
+	/* Unused */
+}
+
+static bool virtio_ntf_do_io_request(struct kvm *kvm, struct ntf_dev *ndev,
struct virt_queue *queue)
+{
+	struct iovec iov[VIRTIO_NOTIFIER_QUEUE_SIZE];
+	unsigned int len = 0;
+	u16 out, in, head;
+	u32 *counters;
+
+	head = virt_queue__get_iov(queue, iov, &out, &in, kvm);
+	counters = iov[0].iov_base;
+
+	printf("\n\n New updated counters from guest:\n\tPanic: %u OOM:
%u\n\n",
+		counters[VIRTIO_NOTIF_PANIC], counters[VIRTIO_NOTIF_OOM]);
+
+	virt_queue__set_used_elem(queue, head, len);
+
+	return true;
+}
+
+static bool virtio_ntf_do_echo_request(struct kvm *kvm, struct ntf_dev *ndev,
struct virt_queue *queue)
+{
+	struct iovec iov[VIRTIO_NOTIFIER_QUEUE_SIZE];
+	u16 out, in, head;
+
+	sleep(1);
+	head = virt_queue__get_iov(queue, iov, &out, &in, kvm);
+	virt_queue__set_used_elem(queue, head, iov[0].iov_len);
+
+	return true;
+}
+
+static void virtio_ntf_do_io(struct kvm *kvm, void *param)
+{
+	struct ntf_dev_job *job	= param;
+	struct virt_queue *vq	= job->vq;
+	struct ntf_dev *ndev	= job->ndev;
+
+	if ((vq - ndev->vqs) == 0) {
+		while (virt_queue__available(vq))
+			virtio_ntf_do_io_request(kvm, ndev, vq);
+	} else {
+		virtio_ntf_do_echo_request(kvm, ndev, vq);
+	}
+
+	ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, vq - ndev->vqs);
+}
+
+static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn)
+{
+	struct ntf_dev *ndev = dev;
+	struct virt_queue *queue;
+	struct ntf_dev_job *job;
+	void *p;
+
+	compat__remove_message(compat_id);
+
+	queue		= &ndev->vqs[vq];
+	queue->pfn	= pfn;
+	p		= guest_pfn_to_host(kvm, queue->pfn);
+
+	job = &ndev->jobs[vq];
+
+	vring_init(&queue->vring, VIRTIO_NOTIFIER_QUEUE_SIZE, p,
VIRTIO_PCI_VRING_ALIGN);
+
+	*job = (struct ntf_dev_job) {
+		.vq	= queue,
+		.ndev	= ndev,
+	};
+
+	thread_pool__init_job(&job->job_id, kvm, virtio_ntf_do_io, job);
+
+	return 0;
+}
+
+static int notify_vq(struct kvm *kvm, void *dev, u32 vq)
+{
+	struct ntf_dev *ndev = dev;
+
+	thread_pool__do_job(&ndev->jobs[vq].job_id);
+
+	return 0;
+}
+
+static int get_pfn_vq(struct kvm *kvm, void *dev, u32 vq)
+{
+	struct ntf_dev *ndev = dev;
+
+	return ndev->vqs[vq].pfn;
+}
+
+static int get_size_vq(struct kvm *kvm, void *dev, u32 vq)
+{
+	return VIRTIO_NOTIFIER_QUEUE_SIZE;
+}
+
+static struct virtio_ops ntf_dev_virtio_ops = (struct virtio_ops) {
+	.set_config		= set_config,
+	.get_config		= get_config,
+	.get_host_features	= get_host_features,
+	.set_guest_features	= set_guest_features,
+	.init_vq		= init_vq,
+	.notify_vq		= notify_vq,
+	.get_pfn_vq		= get_pfn_vq,
+	.get_size_vq		= get_size_vq,
+};
+
+int virtio_notif__init(struct kvm *kvm)
+{
+	struct ntf_dev *ndev;
+	int r;
+
+	ndev = malloc(sizeof(*ndev));
+	if (ndev == NULL)
+		return -ENOMEM;
+
+	r = virtio_init(kvm, ndev, &ndev->vdev, &ntf_dev_virtio_ops,
+			VIRTIO_PCI, PCI_DEVICE_ID_VIRTIO_NOTIFIER, VIRTIO_ID_NOTIFIER,
PCI_CLASS_RNG);
+	if (r < 0)
+		goto cleanup;
+
+	list_add_tail(&ndev->list, &ndevs);
+
+	return 0;
+cleanup:
+	free(ndev);
+
+	return r;
+}
+
+int virtio_notif__exit(struct kvm *kvm)
+{
+	struct ntf_dev *ndev, *tmp;
+
+	list_for_each_entry_safe(ndev, tmp, &ndevs, list) {
+		list_del(&ndev->list);
+		ndev->vdev.ops->exit(kvm, &ndev->vdev);
+		free(ndev);
+	}
+
+	return 0;
+}
-- 
1.7.8.6

On Mon, 23 Jul 2012 22:32:39 +0200, Sasha Levin <levinsasha928 at
gmail.com> wrote:
> As it was discussed recently, there's currently no way for the guest to
notify
> the host about panics. Further more, there's no reasonable way to
notify the
> host of other critical events such as an OOM kill.
I clearly missed the discussion.  Is this actually useful?  In practice,
won't you want the log from the guest?  What makes a virtual guest
different from a physical guest?

Guest watchdog functionality might be useful, but that's simpler to
implement via a virtio watchdog device, and more effective to implement
via a host facility that actually pings guest functionality (rather than
the kernel).

Cheers,
Rusty.

On Mon, Jul 23, 2012 at 10:32:39PM +0200, Sasha Levin
wrote:
> As it was discussed recently, there's currently no way for the guest to
notify
> the host about panics. Further more, there's no reasonable way to
notify the
> host of other critical events such as an OOM kill.
> 
> This short patch series introduces a new device named virtio-notifier which
> does two simple things:
> 
>  1. Provide a simple interface for the guest to notify the host of critical
To get early OOPSes virtio will have to be compiled into the kernel. If
your are so keen on using virtio for this though, why not just use
dedicated virtio serial channel?
>  events. This is easily expandible to add support for any events we may
find
>  interesting for the host to know about.
> 
>  2. Provide an "echo" interface for the host to ping the guest.
This allows
>  the host to ping the guest at intervals chosen by the host, and act
>  accordingly if no response has been received.
> 
> Sasha Levin (2):
>   virtio: Introduce virtio-notifier
>   kvm tools: support virtio-notifier
> 
>  drivers/virtio/Kconfig                  |   11 ++
>  drivers/virtio/Makefile                 |    1 +
>  drivers/virtio/virtio_notifier.c        |  135 ++++++++++++++++++++
>  include/linux/virtio_ids.h              |    1 +
>  include/linux/virtio_notifier.h         |   15 +++
>  tools/kvm/Makefile                      |    1 +
>  tools/kvm/builtin-run.c                 |    6 +
>  tools/kvm/include/kvm/virtio-notifier.h |    9 ++
>  tools/kvm/include/kvm/virtio-pci-dev.h  |    1 +
>  tools/kvm/virtio/notifier.c             |  203
+++++++++++++++++++++++++++++++
>  10 files changed, 383 insertions(+), 0 deletions(-)
>  create mode 100644 drivers/virtio/virtio_notifier.c
>  create mode 100644 include/linux/virtio_notifier.h
>  create mode 100644 tools/kvm/include/kvm/virtio-notifier.h
>  create mode 100644 tools/kvm/virtio/notifier.c
> 
> -- 
> 1.7.8.6
> 
> --
> To unsubscribe from this list: send the line "unsubscribe
linux-kernel" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
--
			Gleb.