Pino Toscano
2020-Jan-28 15:25 UTC
[Libguestfs] [v2v PATCH v2 0/3] Use libosinfo for query device drivers
This patch series integrates libosinfo in virt-v2v to get the list of files for Windows from libosinfo, if possible. The actual data is still from virtio-win, just unpacked. Changes from v1: - adapt to use the priority in libosinfo 1.7.0+ - filter out non-pre-installable drivers - collect all the drivers matching the requirements, not just the first, sorting them by priority like libosinfo does to get the best ones - actually upload the files by the lowercase file name - few minor fixes Pino Toscano (3): build: require libosinfo v2v: add a minimal libosinfo interface v2v: try to get windows driver files from libosinfo m4/guestfs-v2v.m4 | 3 + v2v/Makefile.am | 9 +- v2v/libosinfo-c.c | 237 ++++++++++++++++++++++++++++++++++++++++ v2v/libosinfo.ml | 53 +++++++++ v2v/libosinfo.mli | 48 ++++++++ v2v/libosinfo_utils.ml | 34 ++++++ v2v/libosinfo_utils.mli | 26 +++++ v2v/windows_virtio.ml | 78 ++++++++++++- 8 files changed, 483 insertions(+), 5 deletions(-) create mode 100644 v2v/libosinfo-c.c create mode 100644 v2v/libosinfo.ml create mode 100644 v2v/libosinfo.mli create mode 100644 v2v/libosinfo_utils.ml create mode 100644 v2v/libosinfo_utils.mli -- 2.24.1
Pino Toscano
2020-Jan-28 15:25 UTC
[Libguestfs] [v2v PATCH v2 1/3] build: require libosinfo
Require libosinfo for virt-v2v, as it will be used soon to get more information on guests. --- m4/guestfs-v2v.m4 | 3 +++ v2v/Makefile.am | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/m4/guestfs-v2v.m4 b/m4/guestfs-v2v.m4 index b14c364e..54cdba13 100644 --- a/m4/guestfs-v2v.m4 +++ b/m4/guestfs-v2v.m4 @@ -17,6 +17,9 @@ dnl Virt-v2v. +dnl Check for libosinfo (mandatory) +PKG_CHECK_MODULES([LIBOSINFO], [libosinfo-1.0]) + dnl nbdkit python plugin. AC_MSG_CHECKING([for the nbdkit python plugin name]) AC_ARG_WITH([virt-v2v-nbdkit-python-plugin], diff --git a/v2v/Makefile.am b/v2v/Makefile.am index 64703c36..aae1ffa2 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -190,7 +190,8 @@ virt_v2v_CPPFLAGS = \ -I$(top_srcdir)/lib virt_v2v_CFLAGS = \ $(WARN_CFLAGS) $(WERROR_CFLAGS) \ - $(LIBVIRT_CFLAGS) + $(LIBVIRT_CFLAGS) \ + $(LIBOSINFO_CFLAGS) BOBJECTS = \ $(top_builddir)/common/mlcustomize/firstboot.cmo \ @@ -225,6 +226,7 @@ OCAMLCLIBS = \ $(LIBVIRT_LIBS) \ $(LIBXML2_LIBS) \ $(JANSSON_LIBS) \ + $(LIBOSINFO_LIBS) \ $(LIBINTL) \ -lgnu -- 2.24.1
Pino Toscano
2020-Jan-28 15:25 UTC
[Libguestfs] [v2v PATCH v2 2/3] v2v: add a minimal libosinfo interface
Create a very minimal interface for libosinfo, in particular for OsinfoDb and OsinfoOs, aimed to get the list of device drivers for an OS. --- v2v/Makefile.am | 5 + v2v/libosinfo-c.c | 237 ++++++++++++++++++++++++++++++++++++++++ v2v/libosinfo.ml | 53 +++++++++ v2v/libosinfo.mli | 48 ++++++++ v2v/libosinfo_utils.ml | 34 ++++++ v2v/libosinfo_utils.mli | 26 +++++ 6 files changed, 403 insertions(+) create mode 100644 v2v/libosinfo-c.c create mode 100644 v2v/libosinfo.ml create mode 100644 v2v/libosinfo.mli create mode 100644 v2v/libosinfo_utils.ml create mode 100644 v2v/libosinfo_utils.mli diff --git a/v2v/Makefile.am b/v2v/Makefile.am index aae1ffa2..491238ea 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -63,6 +63,8 @@ SOURCES_MLI = \ input_ova.mli \ input_vmx.mli \ inspect_source.mli \ + libosinfo.mli \ + libosinfo_utils.mli \ libvirt_utils.mli \ linux.mli \ linux_bootloaders.mli \ @@ -112,6 +114,8 @@ SOURCES_ML = \ name_from_disk.ml \ nbdkit.ml \ vCenter.ml \ + libosinfo.ml \ + libosinfo_utils.ml \ libvirt_utils.ml \ DOM.ml \ changeuid.ml \ @@ -164,6 +168,7 @@ SOURCES_ML = \ v2v.ml SOURCES_C = \ + libosinfo-c.c \ qemuopts-c.c # These files are generated and contain *.py embedded as an OCaml string. diff --git a/v2v/libosinfo-c.c b/v2v/libosinfo-c.c new file mode 100644 index 00000000..1f153b3d --- /dev/null +++ b/v2v/libosinfo-c.c @@ -0,0 +1,237 @@ +/* virt-v2v + * Copyright (C) 2020 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/** + * Mini interface to libosinfo. + */ + +#include <config.h> + +#include <osinfo/osinfo.h> + +#include <caml/alloc.h> +#include <caml/callback.h> +#include <caml/fail.h> +#include <caml/memory.h> +#include <caml/mlvalues.h> +#include <caml/custom.h> + +#include <stdio.h> + +#pragma GCC diagnostic ignored "-Wmissing-prototypes" + +#define MAKE_VERSION_HEX(maj, min, mic) \ + (((maj) << 16) | ((min) << 8) | ((mic) << 0)) +#define V2V_LIBOSINFO_VERSION_HEX \ + MAKE_VERSION_HEX(OSINFO_MAJOR_VERSION, OSINFO_MINOR_VERSION, OSINFO_MICRO_VERSION) +#define IS_LIBOSINFO_VERSION(maj, min, mic) \ + V2V_LIBOSINFO_VERSION_HEX >= MAKE_VERSION_HEX(maj, min, mic) + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoFilter, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoLoader, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoList, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OsinfoOsList, g_object_unref) + +typedef OsinfoDb *OsinfoDb_t; +typedef OsinfoOs *OsinfoOs_t; + +/* Wrap and unwrap handles, with a finalizer. */ +#define OsinfoDb_t_val(rv) (*(OsinfoDb_t *)Data_custom_val(rv)) + +#define _OsinfoOs_t_val(rv) (*((OsinfoOs_t *)Data_custom_val(rv))) +#define OsinfoOs_t_val(rv) _OsinfoOs_t_val(Field((rv),0)) + +static void +OsinfoDb_t_finalize (value tv) +{ + OsinfoDb_t t = OsinfoDb_t_val (tv); + if (t) g_object_unref (t); +} + +static struct custom_operations db_custom_operations = { + (char *) "OsinfoDb_t_custom_operations", + OsinfoDb_t_finalize, + custom_compare_default, + custom_hash_default, + custom_serialize_default, + custom_deserialize_default, + custom_compare_ext_default, +}; + +static value +Val_OsinfoDb_t (OsinfoDb_t t) +{ + CAMLparam0 (); + CAMLlocal1 (rv); + + rv = caml_alloc_custom (&db_custom_operations, + sizeof (OsinfoDb_t), 0, 1); + OsinfoDb_t_val(rv) = t; + + CAMLreturn (rv); +} + +static struct custom_operations os_custom_operations = { + (char *) "OsinfoDb_t_custom_operations", + custom_finalize_default, + custom_compare_default, + custom_hash_default, + custom_serialize_default, + custom_deserialize_default, + custom_compare_ext_default, +}; + +static value +Val_OsinfoOs_t (value dbv, OsinfoOs *os) +{ + CAMLparam1 (dbv); + CAMLlocal2 (rv, v); + + v = caml_alloc_custom (&os_custom_operations, + sizeof(OsinfoOs_t), 0, 1); + _OsinfoOs_t_val (v) = os; + rv = caml_alloc_tuple (2); + Store_field (rv, 0, v); + Store_field (rv, 1, dbv); + + CAMLreturn (rv); +} + +value +v2v_osinfo_db_load (value unitv) +{ + CAMLparam1 (unitv); + CAMLlocal1 (rv); + g_autoptr(OsinfoLoader) loader = NULL; + OsinfoDb *db = NULL; + g_autoptr(GError) error = NULL; + + loader = osinfo_loader_new (); + osinfo_loader_process_default_path (loader, &error); + if (error != NULL) + caml_failwith (error->message); + + db = osinfo_loader_get_db (loader); + g_object_ref (db); + + rv = Val_OsinfoDb_t (db); + + CAMLreturn (rv); +} + +value +v2v_osinfo_os_find_os_by_short_id (value dbv, value osv) +{ + CAMLparam2 (dbv, osv); + CAMLlocal1 (rv); + g_autoptr(OsinfoFilter) filter = NULL; + g_autoptr(OsinfoOsList) os_list = NULL; + g_autoptr(OsinfoList) list = NULL; + OsinfoOs *os; + + os_list = osinfo_db_get_os_list (OsinfoDb_t_val (dbv)); + filter = osinfo_filter_new (); + osinfo_filter_add_constraint (filter, OSINFO_PRODUCT_PROP_SHORT_ID, String_val (osv)); + list = osinfo_list_new_filtered (OSINFO_LIST(os_list), filter); + + if (osinfo_list_get_length (list) == 0) + caml_raise_not_found (); + + os = OSINFO_OS(osinfo_list_get_nth (list, 0)); + rv = Val_OsinfoOs_t (dbv, os); + + CAMLreturn (rv); +} + +value +v2v_osinfo_os_get_id (value osv) +{ + CAMLparam1 (osv); + const gchar *id; + + id = osinfo_entity_get_id (OSINFO_ENTITY(OsinfoOs_t_val (osv))); + CAMLreturn (caml_copy_string (id)); +} + +static value +glist_to_value_list (GList *list) +{ + CAMLparam0 (); + CAMLlocal2 (rv, v); + GList *l; + + rv = Val_emptylist; + for (l = list; l != NULL; l = l->next) { + v = caml_alloc (2, 0); + Store_field (v, 0, caml_copy_string (l->data)); + Store_field (v, 1, rv); + rv = v; + } + + CAMLreturn (rv); +} + +value +v2v_osinfo_os_get_device_drivers (value osv) +{ + CAMLparam1 (osv); + CAMLlocal3 (rv, v, vi); + OsinfoDeviceDriverList *list; + gint i, len; + + list = osinfo_os_get_device_drivers (OsinfoOs_t_val (osv)); + len = osinfo_list_get_length (OSINFO_LIST(list)); + + rv = Val_emptylist; + for (i = len - 1; i >= 0; --i) { + OsinfoDeviceDriver *driver; + const gchar *str; + gboolean b; + GList *l; + gint64 i64; + + driver = OSINFO_DEVICE_DRIVER(osinfo_list_get_nth (OSINFO_LIST(list), i)); + + vi = caml_alloc (6, 0); + str = osinfo_device_driver_get_architecture (driver); + Store_field (vi, 0, caml_copy_string (str)); + str = osinfo_device_driver_get_location (driver); + Store_field (vi, 1, caml_copy_string (str)); + b = osinfo_device_driver_get_pre_installable (driver); + Store_field (vi, 2, Val_bool (b)); + b = osinfo_device_driver_get_signed (driver); + Store_field (vi, 3, Val_bool (b)); +#if IS_LIBOSINFO_VERSION(1, 7, 0) + i64 = osinfo_device_driver_get_priority (driver); +#else + /* Same as OSINFO_DEVICE_DRIVER_DEFAULT_PRIORITY in libosinfo 1.7.0+. */ + i64 = 50; +#endif + Store_field (vi, 4, caml_copy_int64 (i64)); + l = osinfo_device_driver_get_files (driver); + Store_field (vi, 5, glist_to_value_list (l)); + g_list_free (l); + + v = caml_alloc (2, 0); + Store_field (v, 0, vi); + Store_field (v, 1, rv); + rv = v; + } + + CAMLreturn (rv); +} diff --git a/v2v/libosinfo.ml b/v2v/libosinfo.ml new file mode 100644 index 00000000..bd9ca126 --- /dev/null +++ b/v2v/libosinfo.ml @@ -0,0 +1,53 @@ +(* virt-v2v + * Copyright (C) 2020 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils +open Tools_utils +open Common_gettext.Gettext + +type osinfo_db_t +type osinfo_os_t + +type osinfo_device_driver = { + architecture : string; + location : string; + pre_installable : bool; + signed : bool; + priority : int64; + files : string list; +} + +external osinfo_os_get_id : osinfo_os_t -> string = "v2v_osinfo_os_get_id" +external osinfo_os_get_device_drivers : osinfo_os_t -> osinfo_device_driver list = "v2v_osinfo_os_get_device_drivers" + +class osinfo_os h + object (self) + method get_id () = osinfo_os_get_id h + method get_device_drivers () = osinfo_os_get_device_drivers h +end + +external osinfo_db_load : unit -> osinfo_db_t = "v2v_osinfo_db_load" +external osinfo_db_find_os_by_short_id : osinfo_db_t -> string -> osinfo_os_t = "v2v_osinfo_os_find_os_by_short_id" + +class osinfo_db () + let h = osinfo_db_load () in + object (self) + method find_os_by_short_id name + let os = osinfo_db_find_os_by_short_id h name in + new osinfo_os os +end diff --git a/v2v/libosinfo.mli b/v2v/libosinfo.mli new file mode 100644 index 00000000..0428ef91 --- /dev/null +++ b/v2v/libosinfo.mli @@ -0,0 +1,48 @@ +(* virt-v2v + * Copyright (C) 2020 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** This module implements a minimal libosinfo API. *) + +type osinfo_os_t + +type osinfo_device_driver = { + architecture : string; + location : string; + pre_installable : bool; + signed : bool; + priority : int64; + files : string list; +} + +class osinfo_os : osinfo_os_t -> object + method get_id : unit -> string + (** Return the ID. *) + method get_device_drivers : unit -> osinfo_device_driver list + (** Return the list of device drivers. *) +end +(** Minimal OsinfoOs wrapper. *) + +class osinfo_db : unit -> object + method find_os_by_short_id : string -> osinfo_os + (** [find_os_by_short_id short-id] get the [osinfo_os] that has the + specified [short-id]. + + Raise [Not_found] in case there is no matching OS. + *) +end +(** Minimal OsinfoDb wrapper. *) diff --git a/v2v/libosinfo_utils.ml b/v2v/libosinfo_utils.ml new file mode 100644 index 00000000..3c7bce10 --- /dev/null +++ b/v2v/libosinfo_utils.ml @@ -0,0 +1,34 @@ +(* virt-v2v + * Copyright (C) 2020 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils +open Tools_utils +open Common_gettext.Gettext + +(* Singleton DB, created on the first access. *) +let db = lazy (new Libosinfo.osinfo_db ()) +(* + * Helper function to get the DB -- use it as sole way to get the DB. + *) +let get_db () + Lazy.force db + +let get_os_by_short_id os + let os = (get_db ())#find_os_by_short_id os in + debug "libosinfo: loaded OS: %s" (os#get_id ()); + os diff --git a/v2v/libosinfo_utils.mli b/v2v/libosinfo_utils.mli new file mode 100644 index 00000000..37485c57 --- /dev/null +++ b/v2v/libosinfo_utils.mli @@ -0,0 +1,26 @@ +(* virt-v2v + * Copyright (C) 2020 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** This module implements helper functions based on libosinfo. *) + +val get_os_by_short_id : string -> Libosinfo.osinfo_os +(** [get_os_by_short_id short-id] get the [Libosinfo.osinfo_os] + that has the specified [short-id]. + + Raise [Not_found] in case there is no matching OS. + *) -- 2.24.1
Pino Toscano
2020-Jan-28 15:25 UTC
[Libguestfs] [v2v PATCH v2 3/3] v2v: try to get windows driver files from libosinfo
Query libosinfo to get the list of files for drivers of Windows guests, copying them if they are local files only. In case it is not possible (e.g. no driver files are listed, or their location is a non-local URL, or they do not exists), the manual filtering of the virtio-win content is done as before. --- v2v/windows_virtio.ml | 78 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/v2v/windows_virtio.ml b/v2v/windows_virtio.ml index 5ec7664b..1468d207 100644 --- a/v2v/windows_virtio.ml +++ b/v2v/windows_virtio.ml @@ -297,10 +297,11 @@ and ddb_regedits inspect drv_name drv_pciid * been copied. *) and copy_drivers g inspect driverdir - [] <> copy_from_virtio_win g inspect "/" driverdir - virtio_iso_path_matches_guest_os - (fun () -> - error (f_"root directory ‘/’ is missing from the virtio-win directory or ISO.\n\nThis should not happen and may indicate that virtio-win or virt-v2v is broken in some way. Please report this as a bug with a full debug log.")) + [] <> copy_from_libosinfo g inspect driverdir || + [] <> copy_from_virtio_win g inspect "/" driverdir + virtio_iso_path_matches_guest_os + (fun () -> + error (f_"root directory ‘/’ is missing from the virtio-win directory or ISO.\n\nThis should not happen and may indicate that virtio-win or virt-v2v is broken in some way. Please report this as a bug with a full debug log.")) and copy_qemu_ga g inspect copy_from_virtio_win g inspect "/" "/" @@ -460,6 +461,75 @@ and virtio_iso_path_matches_qemu_ga path inspect | ("x86_64", "RHEV-QGA64.msi") -> true | _ -> false +(* Look up in libosinfo for the OS, and copy all the locally + * available files specified as drivers for that OS to the [destdir]. + * + * This function does nothing in case either: + * - the osinfo short ID is not found in the libosinfo DB + * - the OS does not have any driver for the same architecture + * - the location of the drivers is a local directory + * + * Files that do not exist are silently skipped. + * + * Returns list of copied files. + *) +and copy_from_libosinfo g inspect destdir + let { i_osinfo = osinfo; i_arch = arch } = inspect in + try + let os = Libosinfo_utils.get_os_by_short_id osinfo in + let drivers = os#get_device_drivers () in + (* + * Filter out drivers that we cannot use: + * - for a different architecture + * - non-pre-installable ones + * - location is an invalid URL, or a non-local one + *) + let drivers + List.filter ( + fun { Libosinfo.architecture; location; pre_installable } -> + if architecture <> arch || not pre_installable then + false + else + try + (match Xml.parse_uri location with + | { Xml.uri_scheme = Some scheme; + Xml.uri_path = Some _ } when scheme = "file" -> true + | _ -> false + ) + with Invalid_argument _ -> false + ) drivers in + (* Sort the drivers by priority, like libosinfo does. *) + let drivers + List.sort ( + fun { Libosinfo.priority = prioA } { Libosinfo.priority = prioB } -> + compare prioB prioA + ) drivers in + (* Any driver available? *) + if drivers = [] then + raise Not_found; + let driver = List.hd drivers in + let uri = Xml.parse_uri driver.Libosinfo.location in + let basedir + match uri.Xml.uri_path with + | Some p -> p + | None -> assert false in + List.filter_map ( + fun f -> + let source = basedir // f in + if not (Sys.file_exists source) then + None + else ( + let target_name = String.lowercase_ascii (Filename.basename f) in + let target = destdir ^ "/" ^ target_name in + debug "windows: copying guest tools bits (via libosinfo): 'host:%s' -> '%s'" + source target; + + g#write target (read_whole_file source); + Some target_name + ) + ) driver.Libosinfo.files + with Not_found -> [] + (* The following function is only exported for unit tests. *) module UNIT_TESTS = struct let virtio_iso_path_matches_guest_os = virtio_iso_path_matches_guest_os -- 2.24.1
Richard W.M. Jones
2020-Feb-03 12:09 UTC
Re: [Libguestfs] [v2v PATCH v2 3/3] v2v: try to get windows driver files from libosinfo
ACK series. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com Fedora Windows cross-compiler. Compile Windows programs, test, and build Windows installers. Over 100 libraries supported. http://fedoraproject.org/wiki/MinGW
Reasonably Related Threads
- [v2v PATCH 3/3] v2v: try to get windows driver files from libosinfo
- [v2v PATCH v2 0/3] Use libosinfo for query device drivers
- [PATCH v3 1/3] v2v: refactor copy_drivers() in Windows_virtio
- [PATCH v4 1/3] v2v: refactor copy_drivers() in Windows_virtio
- [v2v PATCH 0/3] Use libosinfo for query device drivers