I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio The first patch is the kernel module, while the second, the userspace pci device. The host boots with the maximum cpus it should ever use, through the -smp parameter. Due to real machine constraints (which qemu copies), i386 does not allow for any addition of cpus after boot, so this is the most general way. I do however, include an "attempt_buffer" in the userspace part. It's purpose is to allow tools like virt-manager to set a max_cpus (-smp), and a desired number of cpus in their configuration files. (and AFAICT, there's is no easy way for them to tell when the backend driver is up and running) Other than that, it should be pretty much straightforward. Looking forward for your comments
Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
---
drivers/virtio/Kconfig | 6 +
drivers/virtio/Makefile | 1 +
drivers/virtio/virtio_cpu.c | 226 +++++++++++++++++++++++++++++++++++++++++++
drivers/virtio/virtio_pci.c | 1 +
kernel/cpu.c | 2 +
5 files changed, 236 insertions(+), 0 deletions(-)
create mode 100644 drivers/virtio/virtio_cpu.c
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 73a414c..6665d4d 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -23,3 +23,9 @@ config VIRTIO_PCI
If unsure, say M.
+config VIRTIO_CPU
+ tristate "virtio-cpu"
+ depends on VIRTIO_PCI && EXPERIMENTAL
+ ---help---
+ I'll obviously put a better description later
+
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index cc84999..057ebcc 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,3 +1,4 @@
obj-$(CONFIG_VIRTIO) += virtio.o
obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
+obj-$(CONFIG_VIRTIO_CPU) += virtio_cpu.o
diff --git a/drivers/virtio/virtio_cpu.c b/drivers/virtio/virtio_cpu.c
new file mode 100644
index 0000000..8dc0c7d
--- /dev/null
+++ b/drivers/virtio/virtio_cpu.c
@@ -0,0 +1,226 @@
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+
+#include <linux/cpu.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/freezer.h>
+
+#define VIRTIO_ID_CPU 4
+
+#define CMD_CPU_SET 1
+
+struct virtio_hotplug_hdr {
+ u8 cmd;
+ u8 status;
+};
+
+struct hotplug_buf {
+ struct virtio_hotplug_hdr hdr;
+ u32 arg;
+};
+
+struct virtio_hotplug {
+ struct virtio_device *dev;
+ struct virtqueue *cmd_receive;
+ struct task_struct *hotplug_thread;
+ wait_queue_head_t hotplug_wait;
+ u32 hotplug_target;
+ int thread_started;
+};
+
+
+static struct virtio_device_id id_table[] = {
+ { VIRTIO_ID_CPU, VIRTIO_DEV_ANY_ID},
+ { 0 },
+};
+
+void *__alloc_virtio_buf(int size)
+{
+ return kzalloc(size, GFP_KERNEL);
+}
+
+#define alloc_hotplug_buf() (__alloc_virtio_buf(sizeof(struct hotplug_buf)))
+
+static int __send_hotplug_buf(u8 cmd, struct hotplug_buf *buf,
+ struct virtqueue *vq)
+{
+ struct scatterlist sg[2];
+ int err = 0;
+
+ sg_init_table(sg, 2);
+
+ sg_set_buf(&sg[0], &buf->hdr, sizeof(buf->hdr));
+ sg_set_buf(&sg[1], &buf->arg, sizeof(buf->arg));
+
+ err = vq->vq_ops->add_buf(vq, sg, 0, 2, buf);
+ if (err)
+ return err;
+
+ vq->vq_ops->kick(vq);
+
+ return 0;
+}
+
+/* Prepare the virtio driver to receive a command. */
+static int prepare_to_receive(struct virtio_hotplug *v)
+{
+ void *buf = alloc_hotplug_buf();
+
+ if (!buf)
+ return -ENOMEM;
+
+ return __send_hotplug_buf(0, buf, v->cmd_receive);
+}
+
+static void hotplug_add_cpus(struct virtio_hotplug *v)
+{
+ struct sys_device *dev;
+ struct virtqueue *vq = v->cmd_receive;
+ struct hotplug_buf *buf;
+ int i, len;
+ int err;
+
+ while ((buf = vq->vq_ops->get_buf(vq, &len)) != NULL) {
+ switch (buf->hdr.cmd) {
+ case CMD_CPU_SET:
+ v->hotplug_target = buf->arg;
+
+ if (v->hotplug_target > num_present_cpus())
+ v->hotplug_target = num_present_cpus();
+ break;
+ default:
+ printk("%s: Unrecognized command %d\n",
+ __func__, buf->hdr.cmd);
+ break;
+ }
+ }
+ kfree(buf);
+
+
+ while ((i = num_online_cpus()) < v->hotplug_target) {
+ err = cpu_up(i);
+ dev = get_cpu_sysdev(i);
+ if (!err) {
+ if (dev)
+ kobject_uevent(&dev->kobj, KOBJ_ONLINE);
+ }
+ }
+
+ while ((i = num_online_cpus()) > v->hotplug_target) {
+ i--;
+ dev = get_cpu_sysdev(i);
+ err = cpu_down(i);
+ if (!err) {
+ if (dev)
+ kobject_uevent(&dev->kobj, KOBJ_ONLINE);
+ }
+ }
+}
+
+static int virtio_do_hotplug(void *p)
+{
+ struct virtio_hotplug *v = p;
+ DEFINE_WAIT(wait);
+ set_freezable();
+
+ for (;;) {
+ prepare_to_receive(v);
+ prepare_to_wait(&v->hotplug_wait, &wait, TASK_UNINTERRUPTIBLE);
+ v->thread_started = 1;
+ schedule();
+ finish_wait(&v->hotplug_wait, &wait);
+
+ try_to_freeze();
+
+ if (kthread_should_stop())
+ break;
+
+ hotplug_add_cpus(v);
+ v->hotplug_target = 0;
+ }
+ return 0;
+}
+
+/* We need a separate thread because cpu_up() and cpu_down() cannot
+ * be called in interrupt context */
+static bool hotplug_get_cmd(struct virtqueue *vq)
+{
+ struct virtio_hotplug *v = vq->vdev->priv;
+
+ wake_up(&v->hotplug_wait);
+ return true;
+
+}
+
+static int virtio_cpu_probe(struct virtio_device *vdev)
+{
+ int err;
+ struct virtio_hotplug *v;
+
+ v = kzalloc(sizeof(*v), GFP_KERNEL);
+ if (!v)
+ return -ENOMEM;
+
+
+ if (IS_ERR(v->cmd_receive)) {
+ printk("%s: Couldn't find cmd_receive queue\n", __func__);
+ return PTR_ERR(v->cmd_receive);
+ }
+
+ vdev->priv = v;
+
+ init_waitqueue_head(&v->hotplug_wait);
+
+ v->thread_started = 0;
+ v->hotplug_thread = kthread_run(virtio_do_hotplug, v,
+ "virtio_hotplugd\n");
+
+ if (IS_ERR(v->hotplug_thread)) {
+ err = PTR_ERR(v->hotplug_thread);
+ goto out;
+ }
+
+ v->cmd_receive = vdev->config->find_vq(vdev, 0, hotplug_get_cmd);
+
+ printk("Registered virtio hotplug driver\n");
+ return 0;
+out:
+ vdev->config->del_vq(v->cmd_receive);
+ return err;
+}
+
+static void virtio_cpu_remove(struct virtio_device *vdev)
+{
+ struct virtio_hotplug *v = vdev->priv;
+ kthread_stop(v->hotplug_thread);
+ vdev->config->del_vq(v->cmd_receive);
+ kfree(v);
+}
+
+
+static struct virtio_driver virtio_cpu = {
+ .driver.name = KBUILD_MODNAME,
+ .driver.owner = THIS_MODULE,
+ .id_table = id_table,
+ .probe = virtio_cpu_probe,
+ .remove = __devexit_p(virtio_cpu_remove),
+};
+
+static int __init virtio_cpu_init(void)
+{
+ return register_virtio_driver(&virtio_cpu);
+}
+
+static void __exit virtio_cpu_exit(void)
+{
+ unregister_virtio_driver(&virtio_cpu);
+}
+
+module_init(virtio_cpu_init);
+module_exit(virtio_cpu_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Glauber de Oliveira Costa");
+MODULE_DESCRIPTION("virtio driver for cpu hotplugging");
+MODULE_VERSION("1");
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
index e1fa68d..8d5a719 100644
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -67,6 +67,7 @@ static struct pci_device_id virtio_pci_id_table[] = {
{ 0x1AF4, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
{ 0x1AF4, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
{ 0x1AF4, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* Dummy entry */
+ { 0x2523, 0x2325, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, /* cpu hotplug driver */
{ 0 },
};
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 6b3a0c1..c432c28 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -219,6 +219,7 @@ int cpu_down(unsigned int cpu)
mutex_unlock(&cpu_add_remove_lock);
return err;
}
+EXPORT_SYMBOL_GPL(cpu_down);
#endif /*CONFIG_HOTPLUG_CPU*/
/* Requires cpu_add_remove_lock to be held */
@@ -284,6 +285,7 @@ int __cpuinit cpu_up(unsigned int cpu)
mutex_unlock(&cpu_add_remove_lock);
return err;
}
+EXPORT_SYMBOL_GPL(cpu_up);
#ifdef CONFIG_PM_SLEEP_SMP
static cpumask_t frozen_cpus;
--
1.5.0.6
Glauber de Oliveira Costa
2008-Jan-09 02:21 UTC
[PATCH] cpu hotplug driver: userspace back end
Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
---
qemu/Makefile.target | 2 +-
qemu/hw/pc.c | 4 +-
qemu/hw/pc.h | 3 +
qemu/hw/virtio-hotplug.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++
qemu/monitor.c | 11 +++++
qemu/qemu-kvm.h | 2 +
6 files changed, 131 insertions(+), 2 deletions(-)
create mode 100644 qemu/hw/virtio-hotplug.c
diff --git a/qemu/Makefile.target b/qemu/Makefile.target
index bb7be0f..4d5679a 100644
--- a/qemu/Makefile.target
+++ b/qemu/Makefile.target
@@ -464,7 +464,7 @@ VL_OBJS += rtl8139.o
VL_OBJS+= hypercall.o
# virtio devices
-VL_OBJS += virtio.o virtio-net.o virtio-blk.o
+VL_OBJS += virtio.o virtio-net.o virtio-blk.o virtio-hotplug.o
ifeq ($(TARGET_BASE_ARCH), i386)
# Hardware support
diff --git a/qemu/hw/pc.c b/qemu/hw/pc.c
index 3972ab4..c9d8f89 100644
--- a/qemu/hw/pc.c
+++ b/qemu/hw/pc.c
@@ -1029,7 +1029,9 @@ static void pc_init1(ram_addr_t ram_size, int
vga_ram_size,
}
}
-#define USE_HYPERCALL
+ virtio_hotplug_init(pci_bus);
+
+#undef USE_HYPERCALL
#ifdef USE_HYPERCALL
pci_hypercall_init(pci_bus);
#endif
diff --git a/qemu/hw/pc.h b/qemu/hw/pc.h
index 7d1832f..3e2cfb4 100644
--- a/qemu/hw/pc.h
+++ b/qemu/hw/pc.h
@@ -151,6 +151,9 @@ void virtio_net_poll(void);
void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
BlockDriverState *bs);
+/* virtio-hotplug.c */
+void *virtio_hotplug_init(PCIBus *bus);
+
/* extboot.c */
void extboot_init(BlockDriverState *bs, int cmd);
diff --git a/qemu/hw/virtio-hotplug.c b/qemu/hw/virtio-hotplug.c
new file mode 100644
index 0000000..e51f26f
--- /dev/null
+++ b/qemu/hw/virtio-hotplug.c
@@ -0,0 +1,111 @@
+#include "virtio.h"
+#include "net.h"
+#include "pc.h"
+#include "sysemu.h"
+
+typedef struct VirtIOHotplug {
+ VirtIODevice vdev;
+ VirtQueue *cmd_vq;
+ int buffer_ready;
+ struct VirtIOHotplug *next;
+ int job_status;
+} VirtIOHotplug;
+
+typedef struct VirtioHotplugHdr {
+ uint8_t cmd;
+ uint8_t status;
+} VirtioHotplugHdr;
+
+VirtIOHotplug *virtio_hotplug;
+uint32_t attempt_buffer;
+
+#define VIRTIO_ID_HOTPLUG 4 /* arbitrary */
+
+#define CMD_CPU_SET 1
+
+static VirtIOHotplug *to_virtio_hotplug(VirtIODevice *vdev)
+{
+ return (VirtIOHotplug *)vdev;
+}
+
+static void virtio_hotplug_update_config(VirtIODevice *vdev, uint8_t *config)
+{
+ /* nothing to do */
+}
+
+static uint32_t virtio_hotplug_get_features(VirtIODevice *vdev)
+{
+ /* no features yet */
+ return 0;
+}
+
+static void virtio_hotplug_send(VirtIODevice *vdev, VirtQueue *vq, uint8_t cmd,
+ const uint32_t arg)
+{
+ VirtQueueElement elem;
+ VirtioHotplugHdr *hdr;
+ uint32_t *data;
+
+ if (virtqueue_pop(vq, &elem) == 0) {
+ fprintf(stderr, "pop failure\n");
+ return;
+ }
+
+ hdr = (void *)elem.in_sg[0].iov_base;
+ hdr->cmd = cmd;
+
+ data = (int *)elem.in_sg[1].iov_base;
+ *data = arg;
+
+ virtqueue_push(vq, &elem, sizeof(*hdr) + elem.in_sg[1].iov_len);
+ virtio_notify(vdev, vq);
+}
+
+int hotplug_send_cmd(int value)
+{
+
+ if (!virtio_hotplug->buffer_ready) {
+ attempt_buffer = value;
+ return;
+ }
+
+ virtio_hotplug_send(&virtio_hotplug->vdev,
virtio_hotplug->cmd_vq,
+ CMD_CPU_SET, value);
+
+ return 0;
+}
+
+/* TX */
+static void virtio_hotplug_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+{
+ VirtIOHotplug *n = to_virtio_hotplug(vdev);
+
+ n->buffer_ready = 1;
+
+ if (attempt_buffer) {
+ uint32_t value = attempt_buffer;
+ attempt_buffer = 0;
+ hotplug_send_cmd(value);
+ }
+
+}
+
+void *virtio_hotplug_init(PCIBus *bus)
+{
+ VirtIOHotplug *n;
+
+ n = (VirtIOHotplug *)virtio_init_pci(bus, "virtio-hotplug",
0x2523, 0x2325,
+ 0, VIRTIO_ID_HOTPLUG,
+ 0xff, 0x80, 0x00,
+ 6, sizeof(VirtIOHotplug));
+
+ n->vdev.update_config = virtio_hotplug_update_config;
+ n->vdev.get_features = virtio_hotplug_get_features;
+ n->cmd_vq = virtio_add_queue(&n->vdev, 128,
virtio_hotplug_handle_cmd);
+
+ n->buffer_ready = 0;
+
+ virtio_hotplug = n;
+
+ return &n->vdev;
+}
diff --git a/qemu/monitor.c b/qemu/monitor.c
index e03c473..f0c55c6 100644
--- a/qemu/monitor.c
+++ b/qemu/monitor.c
@@ -346,6 +346,16 @@ static void do_cpu_set(int index)
term_printf("Invalid CPU index\n");
}
+static void do_cpu_set_nr(int value)
+{
+ if ((value < 1)) {
+ term_printf("value out of range\n");
+ return;
+ }
+
+ hotplug_send_cmd(value);
+}
+
static void do_info_jit(void)
{
dump_exec_info(NULL, monitor_fprintf);
@@ -1339,6 +1349,7 @@ static term_cmd_t term_cmds[] = {
"", "cancel the current VM migration" },
{ "migrate_set_speed", "s", do_migrate_set_speed,
"value", "set maximum speed (in bytes) for
migrations" },
+ { "cpu_set", "i", do_cpu_set_nr, "value",
"number of cpus in the guest" },
{ NULL, NULL, },
};
diff --git a/qemu/qemu-kvm.h b/qemu/qemu-kvm.h
index e4aeb3a..6bf234d 100644
--- a/qemu/qemu-kvm.h
+++ b/qemu/qemu-kvm.h
@@ -48,6 +48,8 @@ void kvm_tpr_access_report(CPUState *env, uint64_t rip, int
is_write);
int handle_tpr_access(void *opaque, int vcpu,
uint64_t rip, int is_write);
+int hotplug_send_smd(int cpus);
+
#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
#define BITMAP_SIZE(m) (ALIGN(((m)>>TARGET_PAGE_BITS), HOST_LONG_BITS) /
8)
--
1.5.0.6
Glauber de Oliveira Costa wrote:> I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio > The first patch is the kernel module, while the second, the userspace pci device. > > The host boots with the maximum cpus it should ever use, through the -smp parameter. > Due to real machine constraints (which qemu copies), i386 does not allow for any addition > of cpus after boot, so this is the most general way. > > I do however, include an "attempt_buffer" in the userspace part. It's purpose is to > allow tools like virt-manager to set a max_cpus (-smp), and a desired number of cpus > in their configuration files. (and AFAICT, there's is no easy way for them to tell when the > backend driver is up and running) > > Other than that, it should be pretty much straightforward. > > Looking forward for your comments > >I would much prefer to see cpu hotplug implemented via acpi. Such an implementation would work on older kernels without change, and will also work with other operating systems. It isn't a high-speed interface so virtio doesn't buy us anything. Linux appears to support it (CONFIG_ACPI_HOTPLUG_CPU) so all that's needed is the host side support (likely qemu/bios only). Of course hacking on acpi is fun, if you're the kind of person than enjoys dental surgery. -- error compiling committee.c: too many arguments to function
Christian Borntraeger
2008-Jan-09 03:27 UTC
[kvm-devel] [PATCH/RFC 0/2] CPU hotplug virtio driver
Am Mittwoch, 9. Januar 2008 schrieb Glauber de Oliveira Costa:> I'm sending a first draft of my proposed cpu hotplug driver for kvm/virtio > The first patch is the kernel module, while the second, the userspace pcidevice. I personally prefer to use non paravirtualized cpu hotplug and implement the existing interfaces instead in the hypervisor. (sigp/sclp for s390 and maybe acpi for x86 etc.). This is not a performance critical operation. Christian