Libguestfs - Jan 2020 - [v2v PATCH 0/3] Use libosinfo for query device drivers

This patch series integrates libosinfo in virt-v2v to get the list of
files for Windows from libosinfo, if possible. The actual data is still
from virtio-win, just unpacked.

Pino Toscano (3):
  build: require libosinfo
  v2v: add a minimal libosinfo interface
  v2v: try to get windows driver files from libosinfo

 m4/guestfs-v2v.m4       |   3 +
 v2v/Makefile.am         |   9 +-
 v2v/libosinfo-c.c       | 221 ++++++++++++++++++++++++++++++++++++++++
 v2v/libosinfo.ml        |  52 ++++++++++
 v2v/libosinfo.mli       |  47 +++++++++
 v2v/libosinfo_utils.ml  |  34 +++++++
 v2v/libosinfo_utils.mli |  26 +++++
 v2v/windows_virtio.ml   |  63 +++++++++++-
 8 files changed, 450 insertions(+), 5 deletions(-)
 create mode 100644 v2v/libosinfo-c.c
 create mode 100644 v2v/libosinfo.ml
 create mode 100644 v2v/libosinfo.mli
 create mode 100644 v2v/libosinfo_utils.ml
 create mode 100644 v2v/libosinfo_utils.mli

-- 
2.24.1

Require libosinfo for virt-v2v, as it will be used soon to get more
information on guests.
---
 m4/guestfs-v2v.m4 | 3 +++
 v2v/Makefile.am   | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/m4/guestfs-v2v.m4 b/m4/guestfs-v2v.m4
index b14c364e..54cdba13 100644
--- a/m4/guestfs-v2v.m4
+++ b/m4/guestfs-v2v.m4
@@ -17,6 +17,9 @@
 
 dnl Virt-v2v.
 
+dnl Check for libosinfo (mandatory)
+PKG_CHECK_MODULES([LIBOSINFO], [libosinfo-1.0])
+
 dnl nbdkit python plugin.
 AC_MSG_CHECKING([for the nbdkit python plugin name])
 AC_ARG_WITH([virt-v2v-nbdkit-python-plugin],
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 64703c36..aae1ffa2 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -190,7 +190,8 @@ virt_v2v_CPPFLAGS = \
 	-I$(top_srcdir)/lib
 virt_v2v_CFLAGS = \
 	$(WARN_CFLAGS) $(WERROR_CFLAGS) \
-	$(LIBVIRT_CFLAGS)
+	$(LIBVIRT_CFLAGS) \
+	$(LIBOSINFO_CFLAGS)
 
 BOBJECTS = \
 	$(top_builddir)/common/mlcustomize/firstboot.cmo \
@@ -225,6 +226,7 @@ OCAMLCLIBS = \
 	$(LIBVIRT_LIBS) \
 	$(LIBXML2_LIBS) \
 	$(JANSSON_LIBS) \
+	$(LIBOSINFO_LIBS) \
 	$(LIBINTL) \
 	-lgnu
 
-- 
2.24.1

Create a very minimal interface for libosinfo, in particular for
OsinfoDb and OsinfoOs, aimed to get the list of device drivers for an
OS.
---
 v2v/Makefile.am         |   5 +
 v2v/libosinfo-c.c       | 221 ++++++++++++++++++++++++++++++++++++++++
 v2v/libosinfo.ml        |  52 ++++++++++
 v2v/libosinfo.mli       |  47 +++++++++
 v2v/libosinfo_utils.ml  |  34 +++++++
 v2v/libosinfo_utils.mli |  26 +++++
 6 files changed, 385 insertions(+)
 create mode 100644 v2v/libosinfo-c.c
 create mode 100644 v2v/libosinfo.ml
 create mode 100644 v2v/libosinfo.mli
 create mode 100644 v2v/libosinfo_utils.ml
 create mode 100644 v2v/libosinfo_utils.mli

diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index aae1ffa2..491238ea 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -63,6 +63,8 @@ SOURCES_MLI = \
 	input_ova.mli \
 	input_vmx.mli \
 	inspect_source.mli \
+	libosinfo.mli \
+	libosinfo_utils.mli \
 	libvirt_utils.mli \
 	linux.mli \
 	linux_bootloaders.mli \
@@ -112,6 +114,8 @@ SOURCES_ML = \
 	name_from_disk.ml \
 	nbdkit.ml \
 	vCenter.ml \
+	libosinfo.ml \
+	libosinfo_utils.ml \
 	libvirt_utils.ml \
 	DOM.ml \
 	changeuid.ml \
@@ -164,6 +168,7 @@ SOURCES_ML = \
 	v2v.ml
 
 SOURCES_C = \
+	libosinfo-c.c \
 	qemuopts-c.c
 
 # These files are generated and contain *.py embedded as an OCaml string.
diff --git a/v2v/libosinfo-c.c b/v2v/libosinfo-c.c
new file mode 100644
index 00000000..bed2614c
--- /dev/null
+++ b/v2v/libosinfo-c.c
@@ -0,0 +1,221 @@
+/* virt-v2v
+ * Copyright (C) 2020 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/**
+ * Mini interface to libosinfo.
+ */
+
+#include <config.h>
+
+#include <osinfo/osinfo.h>
+
+#include <caml/alloc.h>
+#include <caml/callback.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+#include <caml/custom.h>
+
+#include <stdio.h>
+
+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoFilter, g_object_unref)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoLoader, g_object_unref)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoList, g_object_unref)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoOsList, g_object_unref)
+
+typedef OsinfoDb *OsinfoDb_t;
+typedef OsinfoOs *OsinfoOs_t;
+
+/* Wrap and unwrap handles, with a finalizer. */
+#define OsinfoDb_t_val(rv) (*(OsinfoDb_t *)Data_custom_val(rv))
+
+#define _OsinfoOs_t_val(rv) (*((OsinfoOs_t *)Data_custom_val(rv)))
+#define OsinfoOs_t_val(rv) _OsinfoOs_t_val(Field((rv),0))
+
+static void
+OsinfoDb_t_finalize (value tv)
+{
+  OsinfoDb_t t = OsinfoDb_t_val (tv);
+  if (t) g_object_unref (t);
+}
+
+static struct custom_operations db_custom_operations = {
+  (char *) "OsinfoDb_t_custom_operations",
+  OsinfoDb_t_finalize,
+  custom_compare_default,
+  custom_hash_default,
+  custom_serialize_default,
+  custom_deserialize_default,
+  custom_compare_ext_default,
+};
+
+static value Val_OsinfoDb_t (OsinfoDb_t t)
+{
+  CAMLparam0 ();
+  CAMLlocal1 (rv);
+
+  rv = caml_alloc_custom (&db_custom_operations,
+                          sizeof (OsinfoDb_t), 0, 1);
+  OsinfoDb_t_val(rv) = t;
+
+  CAMLreturn (rv);
+}
+
+static struct custom_operations os_custom_operations = {
+  (char *) "OsinfoDb_t_custom_operations",
+  custom_finalize_default,
+  custom_compare_default,
+  custom_hash_default,
+  custom_serialize_default,
+  custom_deserialize_default,
+  custom_compare_ext_default,
+};
+
+static value Val_OsinfoOs_t (value dbv, OsinfoOs *os)
+{
+  CAMLparam1 (dbv);
+  CAMLlocal2 (rv, v);
+
+  v = caml_alloc_custom (&os_custom_operations,
+                         sizeof(OsinfoOs_t), 0, 1);
+  _OsinfoOs_t_val (v) = os;
+  rv = caml_alloc_tuple (2);
+  Store_field (rv, 0, v);
+  Store_field (rv, 1, dbv);
+
+  CAMLreturn (rv);
+}
+
+value
+v2v_osinfo_db_load (value unitv)
+{
+  CAMLparam1 (unitv);
+  CAMLlocal1 (rv);
+  g_autoptr(OsinfoLoader) loader = NULL;
+  OsinfoDb *db = NULL;
+  g_autoptr(GError) error = NULL;
+
+  loader = osinfo_loader_new ();
+  osinfo_loader_process_default_path (loader, &error);
+  if (error != NULL)
+    caml_failwith (error->message);
+
+  db = osinfo_loader_get_db (loader);
+  g_object_ref (db);
+
+  rv = Val_OsinfoDb_t (db);
+
+  CAMLreturn (rv);
+}
+
+value
+v2v_osinfo_os_find_os_by_short_id (value dbv, value osv)
+{
+  CAMLparam2 (dbv, osv);
+  CAMLlocal1 (rv);
+  g_autoptr(OsinfoFilter) filter = NULL;
+  g_autoptr(OsinfoOsList) os_list = NULL;
+  g_autoptr(OsinfoList) list = NULL;
+  OsinfoOs *os;
+
+  os_list = osinfo_db_get_os_list (OsinfoDb_t_val (dbv));
+  filter = osinfo_filter_new ();
+  osinfo_filter_add_constraint (filter, OSINFO_PRODUCT_PROP_SHORT_ID,
String_val (osv));
+  list = osinfo_list_new_filtered (OSINFO_LIST(os_list), filter);
+
+  if (osinfo_list_get_length (list) == 0)
+    caml_raise_not_found ();
+
+  os = OSINFO_OS(osinfo_list_get_nth (list, 0));
+
+  rv = Val_OsinfoOs_t (dbv, os);
+
+  CAMLreturn (rv);
+}
+
+value
+v2v_osinfo_os_get_id (value osv)
+{
+  CAMLparam1 (osv);
+  const gchar *id;
+
+  id = osinfo_entity_get_id (OSINFO_ENTITY(OsinfoOs_t_val (osv)));
+  CAMLreturn (caml_copy_string (id));
+}
+
+static value
+glist_to_value_list (GList *list)
+{
+  CAMLparam0 ();
+  CAMLlocal2 (rv, v);
+  GList *l;
+
+  rv = Val_emptylist;
+  for (l = list; l != NULL; l = l->next) {
+    v = caml_alloc (2, 0);
+    Store_field (v, 0, caml_copy_string (l->data));
+    Store_field (v, 1, rv);
+    rv = v;
+  }
+
+  CAMLreturn (rv);
+}
+
+value
+v2v_osinfo_os_get_device_drivers (value osv)
+{
+  CAMLparam1 (osv);
+  CAMLlocal3 (rv, v, vi);
+  OsinfoDeviceDriverList *list;
+  gint i, len;
+
+  list = osinfo_os_get_device_drivers (OsinfoOs_t_val (osv));
+  len = osinfo_list_get_length (OSINFO_LIST(list));
+
+  rv = Val_emptylist;
+  for (i = len - 1; i >= 0; --i) {
+    OsinfoDeviceDriver *driver;
+    const gchar *str;
+    gboolean b;
+    GList *l;
+
+    driver = OSINFO_DEVICE_DRIVER(osinfo_list_get_nth (OSINFO_LIST(list), i));
+
+    vi = caml_alloc (5, 0);
+    str = osinfo_device_driver_get_architecture (driver);
+    Store_field (vi, 0, caml_copy_string (str));
+    str = osinfo_device_driver_get_location (driver);
+    Store_field (vi, 1, caml_copy_string (str));
+    b = osinfo_device_driver_get_pre_installable (driver);
+    Store_field (vi, 2, Val_bool (b));
+    b = osinfo_device_driver_get_signed (driver);
+    Store_field (vi, 3, Val_bool (b));
+    l = osinfo_device_driver_get_files (driver);
+    Store_field (vi, 4, glist_to_value_list (l));
+    g_list_free (l);
+
+    v = caml_alloc (2, 0);
+    Store_field (v, 0, vi);
+    Store_field (v, 1, rv);
+    rv = v;
+  }
+
+  CAMLreturn (rv);
+}
diff --git a/v2v/libosinfo.ml b/v2v/libosinfo.ml
new file mode 100644
index 00000000..127c70de
--- /dev/null
+++ b/v2v/libosinfo.ml
@@ -0,0 +1,52 @@
+(* virt-v2v
+ * Copyright (C) 2020 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open Common_gettext.Gettext
+
+type osinfo_db_t
+type osinfo_os_t
+
+type osinfo_device_driver = {
+  architecture : string;
+  location : string;
+  pre_installable : bool;
+  signed : bool;
+  files : string list;
+}
+
+external osinfo_os_get_id : osinfo_os_t -> string =
"v2v_osinfo_os_get_id"
+external osinfo_os_get_device_drivers : osinfo_os_t -> osinfo_device_driver
list = "v2v_osinfo_os_get_device_drivers"
+
+class osinfo_os h +  object (self)
+    method get_id () = osinfo_os_get_id h
+    method get_device_drivers () = osinfo_os_get_device_drivers h
+end
+
+external osinfo_db_load : unit -> osinfo_db_t =
"v2v_osinfo_db_load"
+external osinfo_db_find_os_by_short_id : osinfo_db_t -> string ->
osinfo_os_t = "v2v_osinfo_os_find_os_by_short_id"
+
+class osinfo_db () +  let h = osinfo_db_load () in
+  object (self)
+    method find_os_by_short_id name +      let os =
osinfo_db_find_os_by_short_id h name in
+      new osinfo_os os
+end
diff --git a/v2v/libosinfo.mli b/v2v/libosinfo.mli
new file mode 100644
index 00000000..1b612ee7
--- /dev/null
+++ b/v2v/libosinfo.mli
@@ -0,0 +1,47 @@
+(* virt-v2v
+ * Copyright (C) 2020 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** This module implements a minimal libosinfo API. *)
+
+type osinfo_os_t
+
+type osinfo_device_driver = {
+  architecture : string;
+  location : string;
+  pre_installable : bool;
+  signed : bool;
+  files : string list;
+}
+
+class osinfo_os : osinfo_os_t -> object
+  method get_id : unit -> string
+  (** Return the ID. *)
+  method get_device_drivers : unit -> osinfo_device_driver list
+  (** Return the list of device drivers. *)
+end
+(** Minimal OsinfoOs wrapper. *)
+
+class osinfo_db : unit -> object
+  method find_os_by_short_id : string -> osinfo_os
+  (** [find_os_by_short_id short-id] get the [osinfo_os] that has the
+      specified [short-id].
+
+      Raise [Not_found] in case there is no matching OS.
+   *)
+end
+(** Minimal OsinfoDb wrapper. *)
diff --git a/v2v/libosinfo_utils.ml b/v2v/libosinfo_utils.ml
new file mode 100644
index 00000000..3c7bce10
--- /dev/null
+++ b/v2v/libosinfo_utils.ml
@@ -0,0 +1,34 @@
+(* virt-v2v
+ * Copyright (C) 2020 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open Common_gettext.Gettext
+
+(* Singleton DB, created on the first access. *)
+let db = lazy (new Libosinfo.osinfo_db ())
+(*
+ * Helper function to get the DB -- use it as sole way to get the DB.
+ *)
+let get_db () +  Lazy.force db
+
+let get_os_by_short_id os +  let os = (get_db ())#find_os_by_short_id os in
+  debug "libosinfo: loaded OS: %s" (os#get_id ());
+  os
diff --git a/v2v/libosinfo_utils.mli b/v2v/libosinfo_utils.mli
new file mode 100644
index 00000000..37485c57
--- /dev/null
+++ b/v2v/libosinfo_utils.mli
@@ -0,0 +1,26 @@
+(* virt-v2v
+ * Copyright (C) 2020 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** This module implements helper functions based on libosinfo. *)
+
+val get_os_by_short_id : string -> Libosinfo.osinfo_os
+(** [get_os_by_short_id short-id] get the [Libosinfo.osinfo_os]
+    that has the specified [short-id].
+
+    Raise [Not_found] in case there is no matching OS.
+ *)
-- 
2.24.1

Query libosinfo to get the list of files for drivers of Windows guests,
copying them if they are local files only. In case it is not possible
(e.g. no driver files are listed, or their location is a non-local URL,
or they do not exists), the manual filtering of the virtio-win content
is done as before.
---
 v2v/windows_virtio.ml | 63 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 59 insertions(+), 4 deletions(-)

diff --git a/v2v/windows_virtio.ml b/v2v/windows_virtio.ml
index 5ec7664b..be281247 100644
--- a/v2v/windows_virtio.ml
+++ b/v2v/windows_virtio.ml
@@ -297,10 +297,11 @@ and ddb_regedits inspect drv_name drv_pciid   * been
copied.
  *)
 and copy_drivers g inspect driverdir -  [] <> copy_from_virtio_win g
inspect "/" driverdir
-    virtio_iso_path_matches_guest_os
-    (fun () ->
-      error (f_"root directory ‘/’ is missing from the virtio-win
directory or ISO.\n\nThis should not happen and may indicate that virtio-win or
virt-v2v is broken in some way.  Please report this as a bug with a full debug
log."))
+  [] <> copy_from_libosinfo g inspect driverdir ||
+    [] <> copy_from_virtio_win g inspect "/" driverdir
+      virtio_iso_path_matches_guest_os
+      (fun () ->
+        error (f_"root directory ‘/’ is missing from the virtio-win
directory or ISO.\n\nThis should not happen and may indicate that virtio-win or
virt-v2v is broken in some way.  Please report this as a bug with a full debug
log."))
 
 and copy_qemu_ga g inspect    copy_from_virtio_win g inspect "/"
"/"
@@ -460,6 +461,60 @@ and virtio_iso_path_matches_qemu_ga path inspect    |
("x86_64", "RHEV-QGA64.msi") -> true
   | _ -> false
 
+(* Look up in libosinfo for the OS, and copy all the locally
+ * available files specified as drivers for that OS to the [destdir].
+ *
+ * This function does nothing in case either:
+ * - the osinfo short ID is not found in the libosinfo DB
+ * - the OS does not have any driver for the same architecture
+ * - the location of the drivers is a local directory
+ *
+ * Files that do not exist are silently skipped.
+ *
+ * Returns list of copied files.
+ *)
+and copy_from_libosinfo g inspect destdir +  let { i_osinfo = osinfo; i_arch =
arch } = inspect in
+  try
+    let os = Libosinfo_utils.get_os_by_short_id osinfo in
+    let driver +      List.find (
+        fun { Libosinfo.architecture; location } ->
+          (* Ignore different architectures than the guest's. *)
+          if architecture <> arch then
+            false
+          else
+            try
+              (* Ignore invalid URLs, and non-file ones. *)
+              (match Xml.parse_uri location with
+              | { Xml.uri_scheme = Some scheme;
+                  Xml.uri_path = Some _ } when scheme = "file" ->
true
+              | _ -> false
+              )
+            with Invalid_argument _ -> false
+      ) (os#get_device_drivers ()) in
+    let uri = Xml.parse_uri driver.Libosinfo.location in
+    let dir +      match uri.Xml.uri_path with
+      | Some p -> p
+      | None -> assert false in
+    List.filter_map (
+      fun f ->
+        let source = dir // f in
+        if not (Sys.file_exists source) then
+          None
+        else (
+          let target_name = String.lowercase_ascii (Filename.basename f) in
+          let target = destdir ^ "/" ^ target_name in
+          debug "windows: copying guest tools bits: 'host:%s'
-> '%s'"
+                source target;
+
+          g#copy_in source destdir;
+          Some target_name
+        )
+    ) driver.Libosinfo.files
+  with Not_found -> []
+
 (* The following function is only exported for unit tests. *)
 module UNIT_TESTS = struct
   let virtio_iso_path_matches_guest_os = virtio_iso_path_matches_guest_os
-- 
2.24.1

I couldn't see any issues with the bindings, and the rest looks
fine so ACK series.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v