Libguestfs - Aug 2018 - Using SPDK as QEMU's rootfs disk && A patch for libguestfs to support SPDK

Hi guys,

During the last few weeks, I've been looking at how to use SPDK as
QEMU's
bootable rootfs disk.

I managed to boot up a SPDK rootfs disk by using OVMF UEFI boot-loader. And
in order to deploy the guest OS before start-up(which has a unrecognizable
filesystem to the host), I have written a small patch for the libguestfs.
It was based on Redhat's CentOS libguestfs-1.36.10 RPM, hopefully somebody
could add this SPDK support into the mainline code.

What I'm asking here is a new problem I encountered yesterday. When I
increased the guest memory to more than 8GB, the guest kernel would crash
at boot-up (see attached screen-shot). This issue would not be reproduced
if I only use SPDK as the data disk.

Maybe there is something wrong with the boot-loader?

Thanks, Bob


<0083-Add-SPDK-drive-support.patch>
------------------------------------------------------
>From 7927b0fa080db923989cef7181531dfaa09ccf6c Mon Sep 17 00:00:00 2001
From: Bob Chen <a175818323@gmail.com>
Date: Fri, 27 Jul 2018 12:55:28 +0800
Subject: [PATCH 1/1] Add SPDK drive support

---
 lib/drives.c           | 45 +++++++++++++++++++++++++++++++++++++++++++++
 lib/guestfs-internal.h |  2 ++
 lib/launch-direct.c    | 20 ++++++++++++++++++--
 lib/launch-libvirt.c   |  3 +++
 lib/qemu.c             |  3 +++
 5 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/lib/drives.c b/lib/drives.c
index d3887c1..780b1b2 100644
--- a/lib/drives.c
+++ b/lib/drives.c
@@ -396,6 +396,45 @@ create_drive_iscsi (guestfs_h *g,
 }
 #endif /* DISABLED IN RHEL 7 */

+static struct drive *
+create_drive_spdk (guestfs_h *g,
+                  const struct drive_create_data *data)
+{
+  if (data->username != NULL) {
+    error (g, _("spdk: you cannot specify a username with this
protocol"));
+    return NULL;
+  }
+  if (data->secret != NULL) {
+    error (g, _("spdk: you cannot specify a secret with this
protocol"));
+    return NULL;
+  }
+
+  if (data->nr_servers != 1) {
+    error (g, _("spdk: you must specify exactly one server"));
+    return NULL;
+  }
+
+  if (data->servers[0].transport != drive_transport_unix) {
+    error (g, _("spdk: only unix transport is supported"));
+    return NULL;
+  }
+
+  return create_drive_non_file (g, data);
+}
+
+bool
+guestfs_int_has_spdk_drive (guestfs_h *g)
+{
+  size_t i;
+  struct drive *drv;
+  ITER_DRIVES(g, i, drv) {
+    if (drv->src.protocol == drive_protocol_spdk) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Create the special F</dev/null> drive.
  *
@@ -494,6 +533,7 @@ guestfs_int_drive_protocol_to_string (enum
drive_protocol protocol)
   case drive_protocol_sheepdog: return "sheepdog";
   case drive_protocol_ssh: return "ssh";
   case drive_protocol_tftp: return "tftp";
+  case drive_protocol_spdk: return "spdk";
   }
   abort ();
 }
@@ -878,6 +918,11 @@ guestfs_impl_add_drive_opts (guestfs_h *g, const char
*filename,
     drv = create_drive_curl (g, &data);
   }
 #endif /* DISABLED IN RHEL 7 */
+  else if (STREQ (protocol, "spdk")) {
+    data.protocol = drive_protocol_spdk;
+    data.iface = safe_strdup(g, "vhost-user-scsi-pci");
+    drv = create_drive_spdk(g, &data);
+  }
   else {
     error (g, _("unknown protocol '%s'"), protocol);
     drv = NULL; /*FALLTHROUGH*/
diff --git a/lib/guestfs-internal.h b/lib/guestfs-internal.h
index 3bae02b..a0e5573 100644
--- a/lib/guestfs-internal.h
+++ b/lib/guestfs-internal.h
@@ -199,6 +199,7 @@ enum drive_protocol {
   drive_protocol_sheepdog,
   drive_protocol_ssh,
   drive_protocol_tftp,
+  drive_protocol_spdk,
 };

 enum drive_transport {
@@ -829,6 +830,7 @@ extern void guestfs_int_rollback_drives (guestfs_h *g,
size_t);
 extern void guestfs_int_add_dummy_appliance_drive (guestfs_h *g);
 extern void guestfs_int_free_drives (guestfs_h *g);
 extern const char *guestfs_int_drive_protocol_to_string (enum
drive_protocol protocol);
+extern bool guestfs_int_has_spdk_drive (guestfs_h *g);

 /* appliance.c */
 extern int guestfs_int_build_appliance (guestfs_h *g, char **kernel, char
**initrd, char **appliance);
diff --git a/lib/launch-direct.c b/lib/launch-direct.c
index 3d6e72e..f157324 100644
--- a/lib/launch-direct.c
+++ b/lib/launch-direct.c
@@ -409,8 +409,17 @@ launch_direct (guestfs_h *g, void *datav, const char
*arg)
     ADD_CMDLINE_PRINTF ("%d", g->smp);
   }

-  ADD_CMDLINE ("-m");
-  ADD_CMDLINE_PRINTF ("%d", g->memsize);
+  if (guestfs_int_has_spdk_drive(g)) {
+    ADD_CMDLINE ("-m");
+    ADD_CMDLINE ("1G");
+    ADD_CMDLINE ("-object");
+    ADD_CMDLINE
("memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on");
+    ADD_CMDLINE ("-numa");
+    ADD_CMDLINE ("node,memdev=mem0");
+  } else {
+    ADD_CMDLINE ("-m");
+    ADD_CMDLINE_PRINTF ("%d", g->memsize);
+  }

   /* Force exit instead of reboot on panic */
   ADD_CMDLINE ("-no-reboot");
@@ -567,6 +576,13 @@ launch_direct (guestfs_h *g, void *datav, const char
*arg)
       goto cleanup0;
     }
 #endif
+    else if (drv->iface && STREQ(drv->iface,
"vhost-user-scsi-pci")) {
+      /* SPDK */
+      ADD_CMDLINE ("-chardev");
+      ADD_CMDLINE_PRINTF ("socket,id=vhost,path=%s", escaped_file);
+      ADD_CMDLINE ("-device");
+      ADD_CMDLINE ("vhost-user-scsi-pci,chardev=vhost");
+    }
     else if (drv->iface) {
       ADD_CMDLINE ("-drive");
       ADD_CMDLINE_PRINTF ("%s,if=%s", param, drv->iface);
diff --git a/lib/launch-libvirt.c b/lib/launch-libvirt.c
index 05c398b..468ece9 100644
--- a/lib/launch-libvirt.c
+++ b/lib/launch-libvirt.c
@@ -1548,6 +1548,9 @@ construct_libvirt_xml_disk (guestfs_h *g,
       case drive_protocol_tftp:
         error (g, _("libvirt does not support the qemu curl driver
protocols (ftp, http, etc.); try setting LIBGUESTFS_BACKEND=direct"));
         return -1;
+      case drive_protocol_spdk:
+        error (g, _("libvirt does not support SPDK driver protocol yet;
try setting LIBGUESTFS_BACKEND=direct"));
+        return -1;
       }

       if (construct_libvirt_xml_disk_target (g, xo, drv_index) == -1)
diff --git a/lib/qemu.c b/lib/qemu.c
index 887e31b..a77ea87 100644
--- a/lib/qemu.c
+++ b/lib/qemu.c
@@ -942,6 +942,8 @@ guestfs_int_drive_source_qemu_param (guestfs_h *g,
   case drive_protocol_tftp:
     return make_uri (g, "tftp", src->username, src->secret,
                      &src->servers[0], src->u.exportname);
+  case drive_protocol_spdk:
+    return safe_strdup(g, src->servers[0].u.socket);
   }

   abort ();
@@ -1016,6 +1018,7 @@ guestfs_int_discard_possible (guestfs_h *g, struct
drive *drv,
   case drive_protocol_https:
   case drive_protocol_ssh:
   case drive_protocol_tftp:
+  case drive_protocol_spdk:
     NOT_SUPPORTED (g, -1,
                    _("discard cannot be enabled on this drive: "
                      "protocol '%s' does not support
discard"),
-- 
1.8.3.1