Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 00/12] v2v: Change virt-v2v to use nbdkit for input in several modes.
v3 posted here: https://www.redhat.com/archives/libguestfs/2019-July/msg00200.html v4: - The first patch in the v3 series was just a trivial doc whitespace fix so I pushed it. - There's a new patch using the nbdkit-retry-filter. This is not actually upstream in nbdkit but we know enough about how it will work. - Rebased against master and reran the tests. Rich.
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 01/12] v2v: Factor out the nbdkit VDDK code into a new module.
This refactoring takes the nbdkit-specific code from the ‘-it vddk’
mode and puts it into a separate module. The idea is to reuse this
module (in future commits) to support ssh, curl and rate limiting.
This refactoring is not quite neutral because it moves some of the
prechecking into the Nbdkit.create function. This means it will
happen a little bit later in the cycle (in input#source instead of
input#precheck - refer to the diagram in v2v/types.mli). However it's
still almost in the same place and importantly still happens before we
start copying.
---
v2v/Makefile.am | 2 +
v2v/input_libvirt_vddk.ml | 291 ++++++--------------------------------
v2v/nbdkit.ml | 280 ++++++++++++++++++++++++++++++++++++
v2v/nbdkit.mli | 47 ++++++
4 files changed, 374 insertions(+), 246 deletions(-)
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 84b56d259..2aa4b675e 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -78,6 +78,7 @@ SOURCES_MLI = \
measure_disk.mli \
modules_list.mli \
name_from_disk.mli \
+ nbdkit.mli \
networks.mli \
openstack_image_properties.mli \
output_glance.mli \
@@ -119,6 +120,7 @@ SOURCES_ML = \
var_expander.ml \
python_script.ml \
name_from_disk.ml \
+ nbdkit.ml \
vCenter.ml \
libvirt_utils.ml \
DOM.ml \
diff --git a/v2v/input_libvirt_vddk.ml b/v2v/input_libvirt_vddk.ml
index a8e4fd5ac..1f54ee511 100644
--- a/v2v/input_libvirt_vddk.ml
+++ b/v2v/input_libvirt_vddk.ml
@@ -18,8 +18,6 @@
(** [-i libvirt] when the source is VMware via nbdkit vddk plugin *)
-open Unix
-
open Common_gettext.Gettext
open Tools_utils
open Std_utils
@@ -95,115 +93,16 @@ let parse_input_options options (* Subclass specialized
for handling VMware via nbdkit vddk plugin. *)
class input_libvirt_vddk libvirt_conn input_conn input_password vddk_options
parsed_uri guest - (* The VDDK path. *)
- let libdir - try Some (List.assoc "libdir" vddk_options)
- with Not_found -> None in
-
- (* VDDK libraries are located under lib32/ or lib64/ relative to the
- * libdir. Note this is unrelated to Linux multilib or multiarch.
- *)
- let libNN = sprintf "lib%d" Sys.word_size in
-
- (* Compute the LD_LIBRARY_PATH that we may have to pass to nbdkit. *)
- let library_path = Option.map (fun libdir -> libdir // libNN) libdir in
-
- (* Check that the VDDK path looks reasonable. *)
- let error_unless_vddk_libdir () - (match libdir with
- | None -> ()
- | Some libdir ->
- if not (is_directory libdir) then
- error (f_"‘-io vddk-libdir=%s’ does not point to a directory.
See the virt-v2v-input-vmware(1) manual.") libdir
- );
-
- (match library_path with
- | None -> ()
- | Some library_path ->
- if not (is_directory library_path) then
- error (f_"VDDK library path %s not found or not a directory.
See the virt-v2v-input-vmware(1) manual.") library_path
- )
- in
-
- (* Check that nbdkit is available and new enough. *)
- let error_unless_nbdkit_working () - if 0 <> Sys.command
"nbdkit --version >/dev/null" then
- error (f_"nbdkit is not installed or not working. It is required to
use ‘-it vddk’. See the virt-v2v-input-vmware(1) manual.");
-
- (* Check it's a new enough version. The latest features we
- * require are ‘--exit-with-parent’ and ‘--selinux-label’, both
- * added in 1.1.14. (We use 1.1.16 as the minimum here because
- * it also adds the selinux=yes|no flag in --dump-config).
- *)
- let lines = external_command "nbdkit --help" in
- let lines = String.concat " " lines in
- if String.find lines "exit-with-parent" == -1 ||
- String.find lines "selinux-label" == -1 then
- error (f_"nbdkit is not new enough, you need to upgrade to nbdkit ≥
1.1.16")
- in
-
- (* Check that the VDDK plugin is installed and working *)
- let error_unless_nbdkit_vddk_working () - let set_ld_library_path -
match library_path with
- | None -> ""
- | Some library_path ->
- sprintf "LD_LIBRARY_PATH=%s " (quote library_path) in
-
- let cmd - sprintf "%snbdkit vddk --dump-plugin
>/dev/null"
- set_ld_library_path in
- if Sys.command cmd <> 0 then (
- (* See if we can diagnose why ... *)
- let cmd - sprintf "LANG=C %snbdkit vddk --dump-plugin
2>&1 |
- grep -sq \"cannot open shared object
file\""
- set_ld_library_path in
- let needs_library = Sys.command cmd = 0 in
- if not needs_library then
- error (f_"nbdkit VDDK plugin is not installed or not working. It
is required if you want to use VDDK.
-
-The VDDK plugin is not enabled by default when you compile nbdkit. You have to
read the instructions in the nbdkit sources under ‘plugins/vddk/README.VDDK’ to
find out how to enable the VDDK plugin.
-
-See also the virt-v2v-input-vmware(1) manual.")
- else
- error (f_"nbdkit VDDK plugin is not installed or not working. It
is required if you want to use VDDK.
-
-It looks like you did not set the right path in the ‘-io vddk-libdir’ option,
or your copy of the VDDK directory is incomplete. There should be a library
called ’<libdir>/%s/libvixDiskLib.so.?’.
-
-See also the virt-v2v-input-vmware(1) manual.") libNN
- )
- in
-
let error_unless_thumbprint () if not (List.mem_assoc
"thumbprint" vddk_options) then
error (f_"You must pass the ‘-io vddk-thumbprint’ option with the
SSL thumbprint of the VMware server. To find the thumbprint, see the
nbdkit-vddk-plugin(1) manual. See also the virt-v2v-input-vmware(1)
manual.")
in
- (* Check that nbdkit was compiled with SELinux support (for the
- * --selinux-label option).
- *)
- let error_unless_nbdkit_compiled_with_selinux () - let lines =
external_command "nbdkit --dump-config" in
- (* In nbdkit <= 1.1.15 the selinux attribute was not present
- * at all in --dump-config output so there was no way to tell.
- * Ignore this case because there will be an error later when
- * we try to use the --selinux-label parameter.
- *)
- if List.mem "selinux=no" (List.map String.trim lines) then
- error (f_"nbdkit was compiled without SELinux support. You will
have to recompile nbdkit with libselinux-devel installed, or else set SELinux to
Permissive mode while doing the conversion.")
- in
-
object (self)
inherit input_libvirt libvirt_conn guest as super
method precheck () - error_unless_vddk_libdir ();
- error_unless_nbdkit_working ();
- error_unless_nbdkit_vddk_working ();
- error_unless_thumbprint ();
- if have_selinux then
- error_unless_nbdkit_compiled_with_selinux ()
+ error_unless_thumbprint ()
method as_options let pt_options @@ -231,79 +130,40 @@ object (self)
| None ->
error (f_"<vmware:moref> was not found in the output of
‘virsh dumpxml \"%s\"’. The most likely reason is that libvirt is too
old, try upgrading libvirt to ≥ 3.7.") guest in
- (* Create a temporary directory where we place the sockets. *)
- let tmpdir - let base_dir = (open_guestfs ())#get_cachedir () in
- let t = Mkdtemp.temp_dir ~base_dir "vddk." in
- (* tmpdir must be readable (but not writable) by "other" so
that
- * qemu can open the sockets.
- *)
- chmod t 0o755;
- rmdir_on_exit t;
- t in
-
- (* Start constructing the parts of the incredibly long nbdkit
- * command line which don't change between disks.
+ (* It probably never happens that the server name can be missing
+ * from the libvirt URI, but we need a server name to pass to
+ * nbdkit, so ...
*)
- let args - let add_arg, get_args - let args = ref [] in
- let add_arg a = List.push_front a args in
- let get_args () = List.rev !args in
- add_arg, get_args in
-
- (* It probably never happens that the server name can be missing
- * from the libvirt URI, but we need a server name to pass to
- * nbdkit, so ...
- *)
- let server - match parsed_uri.Xml.uri_server with
- | Some server -> server
- | None ->
- match input_conn with
- | Some input_conn ->
- error (f_"‘-ic %s’ URL does not contain a host name
field")
- input_conn
- | None ->
- error (f_"you must use the ‘-ic’ parameter. See the
virt-v2v-input-vmware(1) manual.") in
-
- (* Similar to above, we also need a username to pass. *)
- let user - match parsed_uri.Xml.uri_user with
- | Some user -> user
- | None -> "root" (* ? *) in
-
- add_arg "nbdkit";
- if verbose () then add_arg "--verbose";
- add_arg "--readonly"; (* important! readonly mode *)
- add_arg "--foreground"; (* run in foreground *)
- add_arg "--exit-with-parent"; (* exit when virt-v2v exits *)
- add_arg "--newstyle"; (* use newstyle NBD protocol *)
- add_arg "--exportname"; add_arg "/";
- if have_selinux then ( (* label the socket so qemu can open it *)
- add_arg "--selinux-label"; add_arg
"system_u:object_r:svirt_socket_t:s0"
- );
-
- (* Name of the plugin. Everything following is a plugin parameter. *)
- add_arg "vddk";
-
- let password_param - match input_password with
- | None ->
- (* nbdkit asks for the password interactively *)
- "password=-"
- | Some password_file ->
- (* nbdkit reads the password from the file *)
- "password=+" ^ password_file in
- add_arg (sprintf "server=%s" server);
- add_arg (sprintf "user=%s" user);
- add_arg password_param;
- add_arg (sprintf "vm=moref=%s" moref);
-
- (* The passthrough parameters. *)
- List.iter (fun (k, v) -> add_arg (sprintf "%s=%s" k v))
vddk_options;
-
- get_args () in
+ let server + match parsed_uri.Xml.uri_server with
+ | Some server -> server
+ | None ->
+ match input_conn with
+ | Some input_conn ->
+ error (f_"‘-ic %s’ URL does not contain a host name
field")
+ input_conn
+ | None ->
+ error (f_"you must use the ‘-ic’ parameter. See the
virt-v2v-input-vmware(1) manual.") in
+
+ let user = parsed_uri.Xml.uri_user in
+
+ let config + try Some (List.assoc "config" vddk_options)
with Not_found -> None in
+ let cookie + try Some (List.assoc "cookie" vddk_options)
with Not_found -> None in
+ let libdir + try Some (List.assoc "libdir" vddk_options)
with Not_found -> None in
+ let nfchostport + try Some (List.assoc "nfchostport"
vddk_options) with Not_found -> None in
+ let port + try Some (List.assoc "port" vddk_options) with
Not_found -> None in
+ let snapshot + try Some (List.assoc "snapshot" vddk_options)
with Not_found -> None in
+ let thumbprint + try List.assoc "thumbprint" vddk_options
+ with Not_found -> assert false (* checked in precheck method *) in
+ let transports + try Some (List.assoc "transports"
vddk_options) with Not_found -> None in
(* Create an nbdkit instance for each disk and rewrite the source
* paths to point to the NBD socket.
@@ -322,79 +182,18 @@ object (self)
* directly as the nbdkit file= parameter, and it is passed
* directly in this form to VDDK.
*)
-
- let sock = tmpdir // sprintf "nbdkit%d.sock" disk.s_disk_id
in
- let qemu_uri = sprintf "nbd:unix:%s:exportname=/" sock in
-
- let pidfile = tmpdir // sprintf "nbdkit%d.pid"
disk.s_disk_id in
-
- (* Construct the final command line with the "static" args
- * above plus the args which vary for each disk.
+ let nbdkit + Nbdkit.create_vddk ?config ?cookie ?libdir
~moref
+ ?nfchostport ?password_file:input_password ?port
+ ~server ?snapshot ~thumbprint ?transports ?user
+ path in
+ let qemu_uri = Nbdkit.run nbdkit in
+
+ (* nbdkit always presents us with the raw disk blocks from
+ * the guest, so force the format to raw here.
*)
- let args - args @ [ "--pidfile"; pidfile;
- "--unix"; sock;
- sprintf "file=%s" path ] in
-
- (* Print the full command we are about to run when debugging. *)
- if verbose () then (
- eprintf "running nbdkit:\n";
- Option.may (eprintf "LD_LIBRARY_PATH=%s") library_path;
- List.iter (fun arg -> eprintf " %s" (quote arg)) args;
- prerr_newline ()
- );
-
- (* Start an nbdkit instance in the background. By using
- * --exit-with-parent we don't have to worry about cleaning
- * it up, hopefully.
- *)
- let args = Array.of_list args in
- let pid = fork () in
- if pid = 0 then (
- (* Child process (nbdkit). *)
- Option.may (putenv "LD_LIBRARY_PATH") library_path;
- execvp "nbdkit" args
- );
-
- (* Wait for the pidfile to appear so we know that nbdkit
- * is listening for requests.
- *)
- if not (wait_for_file pidfile 30) then (
- if verbose () then
- error (f_"nbdkit did not start up. See previous debugging
messages for problems.")
- else
- error (f_"nbdkit did not start up. There may be errors
printed by nbdkit above.
-
-If the messages above are not sufficient to diagnose the problem then add the
‘virt-v2v -v -x’ options and examine the debugging output carefully.")
- );
-
- if have_selinux then (
- (* Note that Unix domain sockets have both a file label and
- * a socket/process label. Using --selinux-label above
- * only set the socket label, but we must also set the file
- * label.
- *)
- ignore (
- run_command ["chcon";
"system_u:object_r:svirt_image_t:s0";
- sock]
- );
- );
- (* ... and the regular Unix permissions, in case qemu is
- * running as another user.
- *)
- chmod sock 0o777;
-
- (* nbdkit from a vddk source always presents us with the raw
- * disk blocks from the guest, so force the format to raw here.
- *)
- { disk with s_qemu_uri = qemu_uri;
- s_format = Some "raw" }
- ) disks in
-
- if verbose () then (
- eprintf "vddk: tmpdir %s:\n%!" tmpdir;
- ignore (Sys.command (sprintf "ls -laZ %s" (quote tmpdir)))
- );
+ { disk with s_qemu_uri = qemu_uri; s_format = Some "raw" }
+ ) disks in
{ source with s_disks = disks }
end
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
new file mode 100644
index 000000000..6bf38daa0
--- /dev/null
+++ b/v2v/nbdkit.ml
@@ -0,0 +1,280 @@
+(* virt-v2v
+ * Copyright (C) 2009-2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Unix
+open Printf
+
+open Common_gettext.Gettext
+open Std_utils
+open Tools_utils
+open Unix_utils
+
+open Utils
+
+type t = {
+ (* The nbdkit plugin name. *)
+ plugin_name : string;
+
+ (* Parameters (includes the plugin name). *)
+ args : string list;
+
+ (* Environment variables that may be needed for nbdkit to work. *)
+ env : (string * string) list;
+}
+
+(* Check that nbdkit is available and new enough. *)
+let error_unless_nbdkit_working () + if 0 <> Sys.command "nbdkit
--version >/dev/null" then
+ error (f_"nbdkit is not installed or not working");
+
+ (* Check it's a new enough version. The latest features we
+ * require are ‘--exit-with-parent’ and ‘--selinux-label’, both
+ * added in 1.1.14. (We use 1.1.16 as the minimum here because
+ * it also adds the selinux=yes|no flag in --dump-config).
+ *)
+ let lines = external_command "nbdkit --help" in
+ let lines = String.concat " " lines in
+ if String.find lines "exit-with-parent" == -1 ||
+ String.find lines "selinux-label" == -1 then
+ error (f_"nbdkit is not new enough, you need to upgrade to nbdkit ≥
1.1.16")
+
+(* Check that nbdkit was compiled with SELinux support (for the
+ * --selinux-label option).
+ *)
+let error_unless_nbdkit_compiled_with_selinux () + if have_selinux then (
+ let lines = external_command "nbdkit --dump-config" in
+ (* In nbdkit <= 1.1.15 the selinux attribute was not present
+ * at all in --dump-config output so there was no way to tell.
+ * Ignore this case because there will be an error later when
+ * we try to use the --selinux-label parameter.
+ *)
+ if List.mem "selinux=no" (List.map String.trim lines) then
+ error (f_"nbdkit was compiled without SELinux support. You will
have to recompile nbdkit with libselinux-devel installed, or else set SELinux to
Permissive mode while doing the conversion.")
+ )
+
+let common_create plugin_name plugin_args plugin_env +
error_unless_nbdkit_working ();
+ error_unless_nbdkit_compiled_with_selinux ();
+
+ (* Start constructing the parts of the incredibly long nbdkit
+ * command line which don't change between disks.
+ *)
+ let add_arg, get_args + let args = ref [] in
+ let add_arg a = List.push_front a args in
+ let get_args () = List.rev !args in
+ add_arg, get_args in
+
+ add_arg "nbdkit";
+ if verbose () then add_arg "--verbose";
+ add_arg "--readonly"; (* important! readonly mode *)
+ add_arg "--foreground"; (* run in foreground *)
+ add_arg "--exit-with-parent"; (* exit when virt-v2v exits *)
+ add_arg "--newstyle"; (* use newstyle NBD protocol *)
+ add_arg "--exportname"; add_arg "/";
+ if have_selinux then ( (* label the socket so qemu can open it *)
+ add_arg "--selinux-label"; add_arg
"system_u:object_r:svirt_socket_t:s0"
+ );
+ let args = get_args () @ [ plugin_name ] @ plugin_args in
+
+ (* Environment. We always add LANG=C. *)
+ let env = ("LANG", "C") :: plugin_env in
+
+ { plugin_name; args; env }
+
+(* VDDK libraries are located under lib32/ or lib64/ relative to the
+ * libdir. Note this is unrelated to Linux multilib or multiarch.
+ *)
+let libNN = sprintf "lib%d" Sys.word_size
+
+(* Create an nbdkit module specialized for reading from VDDK sources. *)
+let create_vddk ?config ?cookie ?libdir ~moref
+ ?nfchostport ?password_file ?port
+ ~server ?snapshot ~thumbprint ?transports ?user path + (*
Compute the LD_LIBRARY_PATH that we may have to pass to nbdkit. *)
+ let ld_library_path = Option.map (fun libdir -> libdir // libNN) libdir in
+
+ (* Check that the VDDK path looks reasonable. *)
+ let error_unless_vddk_libdir () + (match libdir with
+ | None -> ()
+ | Some libdir ->
+ if not (is_directory libdir) then
+ error (f_"‘-io vddk-libdir=%s’ does not point to a directory.
See the virt-v2v-input-vmware(1) manual.") libdir
+ );
+
+ (match ld_library_path with
+ | None -> ()
+ | Some ld_library_path ->
+ if not (is_directory ld_library_path) then
+ error (f_"VDDK library path %s not found or not a directory.
See the virt-v2v-input-vmware(1) manual.") ld_library_path
+ )
+ in
+
+ (* Check that the VDDK plugin is installed and working *)
+ let error_unless_nbdkit_vddk_working () + let set_ld_library_path +
match ld_library_path with
+ | None -> ""
+ | Some ld_library_path ->
+ sprintf "LD_LIBRARY_PATH=%s " (quote ld_library_path) in
+
+ let cmd + sprintf "%snbdkit vddk --dump-plugin
>/dev/null"
+ set_ld_library_path in
+ if Sys.command cmd <> 0 then (
+ (* See if we can diagnose why ... *)
+ let cmd + sprintf "LANG=C %snbdkit vddk --dump-plugin
2>&1 |
+ grep -sq \"cannot open shared object
file\""
+ set_ld_library_path in
+ let needs_library = Sys.command cmd = 0 in
+ if not needs_library then
+ error (f_"nbdkit VDDK plugin is not installed or not working. It
is required if you want to use VDDK.
+
+The VDDK plugin is not enabled by default when you compile nbdkit. You have to
read the instructions in the nbdkit sources under ‘plugins/vddk/README.VDDK’ to
find out how to enable the VDDK plugin.
+
+See also the virt-v2v-input-vmware(1) manual.")
+ else
+ error (f_"nbdkit VDDK plugin is not installed or not working. It
is required if you want to use VDDK.
+
+It looks like you did not set the right path in the ‘-io vddk-libdir’ option,
or your copy of the VDDK directory is incomplete. There should be a library
called ’<libdir>/%s/libvixDiskLib.so.?’.
+
+See also the virt-v2v-input-vmware(1) manual.") libNN
+ )
+ in
+
+ error_unless_vddk_libdir ();
+ error_unless_nbdkit_vddk_working ();
+
+ (* For VDDK we require some user. If it's not supplied, assume root. *)
+ let user = Option.default "root" user in
+
+ let add_arg, get_args + let args = ref [] in
+ let add_arg a = List.push_front a args in
+ let get_args () = List.rev !args in
+ add_arg, get_args in
+
+ let password_param + match password_file with
+ | None ->
+ (* nbdkit asks for the password interactively *)
+ "password=-"
+ | Some password_file ->
+ (* nbdkit reads the password from the file *)
+ "password=+" ^ password_file in
+ add_arg (sprintf "server=%s" server);
+ add_arg (sprintf "user=%s" user);
+ add_arg password_param;
+ add_arg (sprintf "vm=moref=%s" moref);
+ add_arg (sprintf "file=%s" path);
+
+ (* The passthrough parameters. *)
+ Option.may (fun s -> add_arg (sprintf "config=%s" s)) config;
+ Option.may (fun s -> add_arg (sprintf "cookie=%s" s)) cookie;
+ Option.may (fun s -> add_arg (sprintf "libdir=%s" s)) libdir;
+ Option.may (fun s -> add_arg (sprintf "nfchostport=%s" s))
nfchostport;
+ Option.may (fun s -> add_arg (sprintf "port=%s" s)) port;
+ Option.may (fun s -> add_arg (sprintf "snapshot=%s" s))
snapshot;
+ add_arg (sprintf "thumbprint=%s" thumbprint);
+ Option.may (fun s -> add_arg (sprintf "transports=%s" s))
transports;
+
+ let env + match ld_library_path with
+ | None -> []
+ | Some ld_library_path -> ["LD_LIBRARY_PATH", ld_library_path]
in
+
+ common_create "vddk" (get_args ()) env
+
+let run { args; env } + (* Create a temporary directory where we place the
sockets. *)
+ let tmpdir + let base_dir = (open_guestfs ())#get_cachedir () in
+ let t = Mkdtemp.temp_dir ~base_dir "v2vnbdkit." in
+ (* tmpdir must be readable (but not writable) by "other" so that
+ * qemu can open the sockets.
+ *)
+ chmod t 0o755;
+ rmdir_on_exit t;
+ t in
+
+ let id = unique () in
+ let sock = tmpdir // sprintf "nbdkit%d.sock" id in
+ let qemu_uri = sprintf "nbd:unix:%s:exportname=/" sock in
+ let pidfile = tmpdir // sprintf "nbdkit%d.pid" id in
+
+ (* Construct the final command line with the "static" args
+ * above plus the pidfile and socket which vary for each run.
+ *)
+ let args = args @ [ "--pidfile"; pidfile; "--unix"; sock
] in
+
+ (* Print the full command we are about to run when debugging. *)
+ if verbose () then (
+ eprintf "running nbdkit:\n";
+ List.iter (fun (k, v) -> eprintf " %s=%s" k v) env;
+ List.iter (fun arg -> eprintf " %s" (quote arg)) args;
+ prerr_newline ()
+ );
+
+ (* Start an nbdkit instance in the background. By using
+ * --exit-with-parent we don't have to worry about cleaning
+ * it up, hopefully.
+ *)
+ let args = Array.of_list args in
+ let pid = fork () in
+ if pid = 0 then (
+ (* Child process (nbdkit). *)
+ List.iter (fun (k, v) -> putenv k v) env;
+ execvp "nbdkit" args
+ );
+
+ (* Wait for the pidfile to appear so we know that nbdkit
+ * is listening for requests.
+ *)
+ if not (wait_for_file pidfile 30) then (
+ if verbose () then
+ error (f_"nbdkit did not start up. See previous debugging messages
for problems.")
+ else
+ error (f_"nbdkit did not start up. There may be errors printed by
nbdkit above.
+
+If the messages above are not sufficient to diagnose the problem then add the
‘virt-v2v -v -x’ options and examine the debugging output carefully.")
+ );
+
+ if have_selinux then (
+ (* Note that Unix domain sockets have both a file label and
+ * a socket/process label. Using --selinux-label above
+ * only set the socket label, but we must also set the file
+ * label.
+ *)
+ ignore (
+ run_command ["chcon";
"system_u:object_r:svirt_image_t:s0"; sock]
+ );
+ );
+ (* ... and the regular Unix permissions, in case qemu is
+ * running as another user.
+ *)
+ chmod sock 0o777;
+
+ if verbose () then (
+ eprintf "nbdkit: tmpdir %s:\n%!" tmpdir;
+ ignore (Sys.command (sprintf "ls -laZ %s" (quote tmpdir)))
+ );
+
+ qemu_uri
diff --git a/v2v/nbdkit.mli b/v2v/nbdkit.mli
new file mode 100644
index 000000000..3bdec1b56
--- /dev/null
+++ b/v2v/nbdkit.mli
@@ -0,0 +1,47 @@
+(* virt-v2v
+ * Copyright (C) 2009-2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** nbdkit when used as a source. *)
+
+type t
+
+val create_vddk : ?config:string ->
+ ?cookie:string ->
+ ?libdir:string ->
+ moref:string ->
+ ?nfchostport:string ->
+ ?password_file:string ->
+ ?port:string ->
+ server:string ->
+ ?snapshot:string ->
+ thumbprint:string ->
+ ?transports:string ->
+ ?user:string ->
+ string -> t
+(** Create a nbdkit object using the VDDK plugin. The required
+ string parameter is the disk remote path.
+
+ This can fail (calling [error]) for a variety of reasons, such
+ as nbdkit not being available, wrong version, missing plugin, etc.
+
+ Note this doesn't run nbdkit yet, it just creates the object. *)
+
+val run : t -> string
+(** Start running nbdkit.
+
+ Returns the QEMU URI that you can use to connect to this instance. *)
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 02/12] v2v: Generic code for querying nbdkit version and plugin.
In forthcoming commits we will be adding support for ssh, curl and
other features that require nbdkit >= 1.12.
As a prelude to that work, add generic code for querying ‘nbdkit
--dump-config’ and ‘nbdkit plugin --dump-plugin’ and checking the
minimum version number.
This changes the minimum version from 1.1.16 to 1.2, although that was
released about a year ago and is widely available, and in any case
we're going to require 1.12 in the next commit.
---
v2v/nbdkit.ml | 112 ++++++++++++++++++++++++++++++++------------------
1 file changed, 72 insertions(+), 40 deletions(-)
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index 6bf38daa0..8ae6549e9 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -26,6 +26,9 @@ open Unix_utils
open Utils
+let nbdkit_min_version = (1, 2)
+let nbdkit_min_version_string = "1.2"
+
type t = {
(* The nbdkit plugin name. *)
plugin_name : string;
@@ -35,42 +38,76 @@ type t = {
(* Environment variables that may be needed for nbdkit to work. *)
env : (string * string) list;
+
+ (* nbdkit --dump-config output. *)
+ dump_config : (string * string) list;
+
+ (* nbdkit plugin_name --dump-plugin output. *)
+ dump_plugin : (string * string) list;
}
(* Check that nbdkit is available and new enough. *)
let error_unless_nbdkit_working () if 0 <> Sys.command "nbdkit
--version >/dev/null" then
- error (f_"nbdkit is not installed or not working");
+ error (f_"nbdkit is not installed or not working")
+
+(* Check that nbdkit is at or above the minimum version. *)
+let re_major_minor = PCRE.compile "(\\d+)\\.(\\d+)"
+
+let error_unless_nbdkit_min_version dump_config + let version + let version
+ try List.assoc "version" dump_config
+ with Not_found ->
+ error (f_"nbdkit --dump-config did not print version. This might
be a very old or broken nbdkit binary.") in
+ debug "nbdkit version: %s" version;
+ if PCRE.matches re_major_minor version then
+ (int_of_string (PCRE.sub 1), int_of_string (PCRE.sub 2))
+ else
+ error (f_"nbdkit --dump-config: could not parse version: %s")
version in
- (* Check it's a new enough version. The latest features we
- * require are ‘--exit-with-parent’ and ‘--selinux-label’, both
- * added in 1.1.14. (We use 1.1.16 as the minimum here because
- * it also adds the selinux=yes|no flag in --dump-config).
- *)
- let lines = external_command "nbdkit --help" in
- let lines = String.concat " " lines in
- if String.find lines "exit-with-parent" == -1 ||
- String.find lines "selinux-label" == -1 then
- error (f_"nbdkit is not new enough, you need to upgrade to nbdkit ≥
1.1.16")
+ if version < nbdkit_min_version then
+ error (f_"nbdkit is too old. nbdkit >= %s is required.")
+ nbdkit_min_version_string
(* Check that nbdkit was compiled with SELinux support (for the
* --selinux-label option).
*)
-let error_unless_nbdkit_compiled_with_selinux () +let
error_unless_nbdkit_compiled_with_selinux dump_config if have_selinux then (
- let lines = external_command "nbdkit --dump-config" in
- (* In nbdkit <= 1.1.15 the selinux attribute was not present
- * at all in --dump-config output so there was no way to tell.
- * Ignore this case because there will be an error later when
- * we try to use the --selinux-label parameter.
- *)
- if List.mem "selinux=no" (List.map String.trim lines) then
+ let selinux = try List.assoc "selinux" dump_config with Not_found
-> "no" in
+ if selinux = "no" then
error (f_"nbdkit was compiled without SELinux support. You will
have to recompile nbdkit with libselinux-devel installed, or else set SELinux to
Permissive mode while doing the conversion.")
)
let common_create plugin_name plugin_args plugin_env
error_unless_nbdkit_working ();
- error_unless_nbdkit_compiled_with_selinux ();
+
+ (* Environment. We always add LANG=C. *)
+ let env = ("LANG", "C") :: plugin_env in
+ let env_as_string + String.concat " " (List.map (fun (k, v)
-> sprintf "%s=%s" k (quote v))
+ env) in
+
+ (* Get the nbdkit --dump-config output and check minimum
+ * required version of nbdkit.
+ *)
+ let dump_config + let lines + external_command (sprintf "%s
nbdkit --dump-config" env_as_string) in
+ List.map (String.split "=") lines in
+
+ error_unless_nbdkit_min_version dump_config;
+ error_unless_nbdkit_compiled_with_selinux dump_config;
+
+ (* Get the nbdkit plugin_name --dump-plugin output, which also
+ * checks that the plugin is available and loadable.
+ *)
+ let dump_plugin + let lines + external_command (sprintf "%s
nbdkit %s --dump-plugin"
+ env_as_string plugin_name) in
+ List.map (String.split "=") lines in
(* Start constructing the parts of the incredibly long nbdkit
* command line which don't change between disks.
@@ -93,10 +130,7 @@ let common_create plugin_name plugin_args plugin_env );
let args = get_args () @ [ plugin_name ] @ plugin_args in
- (* Environment. We always add LANG=C. *)
- let env = ("LANG", "C") :: plugin_env in
-
- { plugin_name; args; env }
+ { plugin_name; args; env; dump_config; dump_plugin }
(* VDDK libraries are located under lib32/ or lib64/ relative to the
* libdir. Note this is unrelated to Linux multilib or multiarch.
@@ -109,6 +143,10 @@ let create_vddk ?config ?cookie ?libdir ~moref
~server ?snapshot ~thumbprint ?transports ?user path (*
Compute the LD_LIBRARY_PATH that we may have to pass to nbdkit. *)
let ld_library_path = Option.map (fun libdir -> libdir // libNN) libdir in
+ let env + match ld_library_path with
+ | None -> []
+ | Some ld_library_path -> ["LD_LIBRARY_PATH", ld_library_path]
in
(* Check that the VDDK path looks reasonable. *)
let error_unless_vddk_libdir () @@ -127,23 +165,22 @@ let create_vddk ?config
?cookie ?libdir ~moref
)
in
- (* Check that the VDDK plugin is installed and working *)
+ (* Check that the VDDK plugin is installed and working. We also
+ * check this later when calling common_create, but this version
+ * has better troubleshooting output.
+ *)
let error_unless_nbdkit_vddk_working () - let set_ld_library_path -
match ld_library_path with
- | None -> ""
- | Some ld_library_path ->
- sprintf "LD_LIBRARY_PATH=%s " (quote ld_library_path) in
-
+ let env_as_string + String.concat " " (List.map (fun (k, v)
-> sprintf "%s=%s" k (quote v))
+ env) in
let cmd - sprintf "%snbdkit vddk --dump-plugin
>/dev/null"
- set_ld_library_path in
+ sprintf "%s nbdkit vddk --dump-plugin >/dev/null"
env_as_string in
if Sys.command cmd <> 0 then (
(* See if we can diagnose why ... *)
let cmd - sprintf "LANG=C %snbdkit vddk --dump-plugin
2>&1 |
+ sprintf "LANG=C %s nbdkit vddk --dump-plugin 2>&1 |
grep -sq \"cannot open shared object
file\""
- set_ld_library_path in
+ env_as_string in
let needs_library = Sys.command cmd = 0 in
if not needs_library then
error (f_"nbdkit VDDK plugin is not installed or not working. It
is required if you want to use VDDK.
@@ -196,11 +233,6 @@ See also the virt-v2v-input-vmware(1) manual.") libNN
add_arg (sprintf "thumbprint=%s" thumbprint);
Option.may (fun s -> add_arg (sprintf "transports=%s" s))
transports;
- let env - match ld_library_path with
- | None -> []
- | Some ld_library_path -> ["LD_LIBRARY_PATH", ld_library_path]
in
-
common_create "vddk" (get_args ()) env
let run { args; env } --
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 03/12] v2v: nbdkit: Add support for nbdkit-ssh-plugin.
---
v2v/nbdkit.ml | 26 ++++++++++++++++++++++++++
v2v/nbdkit.mli | 18 ++++++++++++++++++
2 files changed, 44 insertions(+)
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index 8ae6549e9..44fe0e8e7 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -29,6 +29,11 @@ open Utils
let nbdkit_min_version = (1, 2)
let nbdkit_min_version_string = "1.2"
+type password +| NoPassword (* no password option at all *)
+| AskForPassword (* password=- *)
+| PasswordFile of string (* password=+file *)
+
type t = {
(* The nbdkit plugin name. *)
plugin_name : string;
@@ -235,6 +240,27 @@ See also the virt-v2v-input-vmware(1) manual.") libNN
common_create "vddk" (get_args ()) env
+(* Create an nbdkit module specialized for reading from SSH sources. *)
+let create_ssh ~password ?port ~server ?user path + let add_arg, get_args +
let args = ref [] in
+ let add_arg a = List.push_front a args in
+ let get_args () = List.rev !args in
+ add_arg, get_args in
+
+ add_arg (sprintf "host=%s" server);
+ Option.may (fun s -> add_arg (sprintf "port=%s" s)) port;
+ Option.may (fun s -> add_arg (sprintf "user=%s" s)) user;
+ (match password with
+ | NoPassword -> ()
+ | AskForPassword -> add_arg "password=-"
+ | PasswordFile password_file ->
+ add_arg (sprintf "password=+%s" password_file)
+ );
+ add_arg (sprintf "path=%s" path);
+
+ common_create "ssh" (get_args ()) []
+
let run { args; env } (* Create a temporary directory where we place the
sockets. *)
let tmpdir diff --git a/v2v/nbdkit.mli b/v2v/nbdkit.mli
index 3bdec1b56..36faff03b 100644
--- a/v2v/nbdkit.mli
+++ b/v2v/nbdkit.mli
@@ -41,6 +41,24 @@ val create_vddk : ?config:string ->
Note this doesn't run nbdkit yet, it just creates the object. *)
+type password +| NoPassword
+| AskForPassword
+| PasswordFile of string
+
+val create_ssh : password:password ->
+ ?port:string ->
+ server:string ->
+ ?user:string ->
+ string -> t
+(** Create a nbdkit object using the SSH plugin. The required
+ string parameter is the remote path.
+
+ This can fail (calling [error]) for a variety of reasons, such
+ as nbdkit not being available, wrong version, missing plugin, etc.
+
+ Note this doesn't run nbdkit yet, it just creates the object. *)
+
val run : t -> string
(** Start running nbdkit.
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 04/12] v2v: xen: Replace qemu block ssh driver with nbdkit-ssh-plugin.
Initially this is a like-for-like replacement, but in future commits
this will allow us to implement:
- password authentication (instead of SSH agent)
- bandwidth throttling
- readahead
Note this requires nbdkit >= 1.12.
---
docs/guestfs-building.pod | 5 +++-
v2v/input_libvirt_xen_ssh.ml | 48 +++++++++---------------------------
v2v/nbdkit.ml | 4 +--
3 files changed, 17 insertions(+), 40 deletions(-)
diff --git a/docs/guestfs-building.pod b/docs/guestfs-building.pod
index 441fe0937..96a301a91 100644
--- a/docs/guestfs-building.pod
+++ b/docs/guestfs-building.pod
@@ -268,10 +268,13 @@ Optional. Used only for testing.
=item qemu-nbd
-=item nbdkit
+=item nbdkit E<ge> 1.12
Optional. qemu-nbd is used for testing.
+L<virt-v2v(1)> requires nbdkit E<ge> 1.12 for various input and
output
+modes.
+
=item uml_mkcow
Optional. For the L<UML backend|guestfs(3)/BACKEND>.
diff --git a/v2v/input_libvirt_xen_ssh.ml b/v2v/input_libvirt_xen_ssh.ml
index ab6d933bc..9a941d070 100644
--- a/v2v/input_libvirt_xen_ssh.ml
+++ b/v2v/input_libvirt_xen_ssh.ml
@@ -45,18 +45,15 @@ object (self)
let source, disks, _ = parse_libvirt_domain self#conn guest in
+ let port + match parsed_uri.uri_port with
+ | 0 | 22 -> None
+ | i -> Some (string_of_int i) in
+
+ let user = parsed_uri.uri_user in
+
(* Map the <source/> filename (which is relative to the remote
- * Xen server) to an ssh URI. This is a JSON URI looking something
- * like this:
- *
- * json: {
- * "file.driver": "ssh",
- * "file.user": "username",
- * "file.host": "xen-host",
- * "file.port": 1022,
- * "file.path": <remote-path>,
- * "file.host_key_check": "no"
- * }
+ * Xen server) to an ssh URI pointing to nbdkit.
*)
let disks = List.map (
function
@@ -64,32 +61,9 @@ object (self)
disk
| { p_source_disk = disk; p_source = P_source_dev path }
| { p_source_disk = disk; p_source = P_source_file path } ->
- (* Construct the JSON parameters. *)
- let json_params = [
- "file.driver", JSON.String "ssh";
- "file.path", JSON.String path;
- "file.host", JSON.String server;
- "file.host_key_check", JSON.String "no";
- ] in
-
- let json_params - match parsed_uri.uri_port with
- | 0 | 22 -> json_params
- (* qemu will actually assert-fail if you send the port
- * number as a string ...
- *)
- | i -> ("file.port", JSON.Int (Int64.of_int i)) ::
json_params in
-
- let json_params - match parsed_uri.uri_user with
- | None -> json_params
- | Some user -> ("file.user", JSON.String user) ::
json_params in
-
- debug "ssh: json parameters: %s" (JSON.string_of_doc
json_params);
-
- (* Turn the JSON parameters into a 'json:' protocol string. *)
- let qemu_uri = "json: " ^ JSON.string_of_doc json_params in
-
+ let nbdkit = Nbdkit.create_ssh ~password:NoPassword
+ ?port ~server ?user path in
+ let qemu_uri = Nbdkit.run nbdkit in
{ disk with s_qemu_uri = qemu_uri }
) disks in
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index 44fe0e8e7..d21c862b3 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -26,8 +26,8 @@ open Unix_utils
open Utils
-let nbdkit_min_version = (1, 2)
-let nbdkit_min_version_string = "1.2"
+let nbdkit_min_version = (1, 12)
+let nbdkit_min_version_string = "1.12"
type password | NoPassword (* no password option at all *)
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 05/12] v2v: -i vmx -it ssh: Replace qemu block ssh driver with nbdkit-ssh-plugin.
One immediately advantage is we can use libvirt again.
---
v2v/input_vmx.ml | 34 ++++++++--------------------------
1 file changed, 8 insertions(+), 26 deletions(-)
diff --git a/v2v/input_vmx.ml b/v2v/input_vmx.ml
index e3469308d..0db1945ea 100644
--- a/v2v/input_vmx.ml
+++ b/v2v/input_vmx.ml
@@ -236,24 +236,14 @@ and qemu_uri_of_filename vmx_source filename if
remote_file_exists uri flat_vmdk then (flat_vmdk, "raw")
else (abs_path, format) in
- let json_params = [
- "file.driver", JSON.String "ssh";
- "file.path", JSON.String abs_path;
- "file.host", JSON.String (server_of_uri uri);
- "file.host_key_check", JSON.String "no";
- ] in
- let json_params - match uri.Xml.uri_user with
- | None -> json_params
- | Some user ->
- ("file.user", JSON.String user) :: json_params in
- let json_params - match port_of_uri uri with
- | None -> json_params
- | Some port ->
- ("file.port", JSON.Int (Int64.of_int port)) :: json_params
in
-
- "json:" ^ JSON.string_of_doc json_params, format
+ let server = server_of_uri uri in
+ let port = Option.map string_of_int (port_of_uri uri) in
+ let user = uri.Xml.uri_user in
+
+ let nbdkit = Nbdkit.create_ssh ~password:NoPassword ~server
+ ?port ?user abs_path in
+ let qemu_uri = Nbdkit.run nbdkit in
+ qemu_uri, format
and absolute_path_from_other_file other_filename filename if not
(Filename.is_relative filename) then filename
@@ -402,14 +392,6 @@ object
method as_options = "-i vmx " ^ arg
- method precheck () - match input_transport with
- | None -> ()
- | Some `SSH ->
- if backend_is_libvirt () then
- error (f_"because libvirtd doesn't pass the SSH_AUTH_SOCK
environment variable to qemu you must set this environment variable:\n\nexport
LIBGUESTFS_BACKEND=direct\n\nand then rerun the virt-v2v command.");
- error_if_no_ssh_agent ()
-
method source () let vmx_source = vmx_source_of_arg input_transport arg
in
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 06/12] v2v: nbdkit: Add support for nbdkit-curl-plugin.
---
v2v/nbdkit.ml | 23 +++++++++++++++++++++++
v2v/nbdkit.mli | 13 +++++++++++++
2 files changed, 36 insertions(+)
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index d21c862b3..4bbb8f043 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -261,6 +261,29 @@ let create_ssh ~password ?port ~server ?user path
common_create "ssh" (get_args ()) []
+(* Create an nbdkit module specialized for reading from Curl sources. *)
+let create_curl ?cookie ~password ?(sslverify=true) ?user url + let add_arg,
get_args + let args = ref [] in
+ let add_arg a = List.push_front a args in
+ let get_args () = List.rev !args in
+ add_arg, get_args in
+
+ Option.may (fun s -> add_arg (sprintf "user=%s" s)) user;
+ (match password with
+ | NoPassword -> ()
+ | AskForPassword -> add_arg "password=-"
+ | PasswordFile password_file ->
+ add_arg (sprintf "password=+%s" password_file)
+ );
+ (* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *)
+ add_arg "timeout=2000";
+ Option.may (fun s -> add_arg (sprintf "cookie=%s" s)) cookie;
+ if not sslverify then add_arg "sslverify=false";
+ add_arg (sprintf "url=%s" url);
+
+ common_create "curl" (get_args ()) []
+
let run { args; env } (* Create a temporary directory where we place the
sockets. *)
let tmpdir diff --git a/v2v/nbdkit.mli b/v2v/nbdkit.mli
index 36faff03b..efbb62f98 100644
--- a/v2v/nbdkit.mli
+++ b/v2v/nbdkit.mli
@@ -59,6 +59,19 @@ val create_ssh : password:password ->
Note this doesn't run nbdkit yet, it just creates the object. *)
+val create_curl : ?cookie:string ->
+ password:password ->
+ ?sslverify:bool ->
+ ?user:string ->
+ string -> t
+(** Create a nbdkit object using the Curl plugin. The required
+ string parameter is the URL.
+
+ This can fail (calling [error]) for a variety of reasons, such
+ as nbdkit not being available, wrong version, missing plugin, etc.
+
+ Note this doesn't run nbdkit yet, it just creates the object. *)
+
val run : t -> string
(** Start running nbdkit.
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 07/12] v2v: nbdkit: Add the readahead filter unconditionally if it is available.
The readahead filter is a self-configuring filter that makes
sequential reads faster when the plugin is slow (and all of the
plugins we use here are always slow).
I observed the behaviour of the readahead filter with our qcow2
overlay when converting a guest from a vCenter source. Even when
doing random reads, qemu issues 64K reads which happen to also be the
minimum request size of the readahead filter, so there is no extra
overhead. When doing the sequential copy the readahead filter
performed better than qemu curl’s readahead because it scaled the
prefetched data appropriately on long contiguous stretches and then
shrunk it back to 64K around fragmented parts of the base image.
---
v2v/nbdkit.ml | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index 4bbb8f043..b2d77f963 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -49,6 +49,9 @@ type t = {
(* nbdkit plugin_name --dump-plugin output. *)
dump_plugin : (string * string) list;
+
+ (* nbdkit directory containing the filters. *)
+ filterdir : string;
}
(* Check that nbdkit is available and new enough. *)
@@ -105,6 +108,12 @@ let common_create plugin_name plugin_args plugin_env
error_unless_nbdkit_min_version dump_config;
error_unless_nbdkit_compiled_with_selinux dump_config;
+ (* Get the filterdir. *)
+ let filterdir + try List.assoc "filterdir" dump_config
+ with Not_found ->
+ error (f_"nbdkit --dump-config output did not contain
filterdir") in
+
(* Get the nbdkit plugin_name --dump-plugin output, which also
* checks that the plugin is available and loadable.
*)
@@ -133,9 +142,17 @@ let common_create plugin_name plugin_args plugin_env if
have_selinux then ( (* label the socket so qemu can open it *)
add_arg "--selinux-label"; add_arg
"system_u:object_r:svirt_socket_t:s0"
);
+
+ (* Adding the readahead filter is always a win for our access
+ * patterns. However if it doesn't exist don't worry.
+ *)
+ if Sys.file_exists (filterdir // "nbdkit-readahead-filter.so") then
(
+ add_arg "--filter"; add_arg "readahead"
+ );
+
let args = get_args () @ [ plugin_name ] @ plugin_args in
- { plugin_name; args; env; dump_config; dump_plugin }
+ { plugin_name; args; env; dump_config; dump_plugin; filterdir }
(* VDDK libraries are located under lib32/ or lib64/ relative to the
* libdir. Note this is unrelated to Linux multilib or multiarch.
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 08/12] v2v: -i libvirtxml: Replace qemu block curl driver with nbdkit-curl-plugin.
‘virt-v2v -i libvirtxml’ has a little-known feature where it can read
network disks over HTTP or HTTPS. This can be used to test VMware
conversions without needing VMware, although as far as I can tell the
current test suite does not use the feature. This commit changes this
functionality to use nbdkit-curl-plugin instead of the qemu curl
driver.
This change shouldn't affect functionality. The readahead size is
changed from a fixed 1M buffer to using the readahead filter which is
self-configuring.
See also commit 38bf2a0f97c2e814d28c447ff6856bdd2d68df36 which
originally introduced this functionality in 2017.
---
v2v/parse_libvirt_xml.ml | 38 ++++++++++++--------------------------
1 file changed, 12 insertions(+), 26 deletions(-)
diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml
index 95273c89c..97d8a5cd8 100644
--- a/v2v/parse_libvirt_xml.ml
+++ b/v2v/parse_libvirt_xml.ml
@@ -46,31 +46,6 @@ let get_drive_slot str offset warning (f_"could
not parse device name ‘%s’ from the source libvirt XML") str;
None
-(* Create a JSON URI for qemu referring to a remote CURL (http/https)
- * resource. See also [v2v/vCenter.ml].
- *)
-let create_curl_qemu_uri driver host port path - let url - let port -
match driver, port with
- | _, None -> ""
- | "https", Some 443 -> ""
- | "http", Some 80 -> ""
- | _, Some port when port >= 1 -> ":" ^ string_of_int port
- | _, Some port -> invalid_arg "invalid port number in libvirt
XML" in
- sprintf "%s://%s%s%s" driver host port (uri_quote path) in
-
- let json_params = [
- "file.driver", JSON.String driver; (* "http" or
"https" *)
- "file.url", JSON.String url;
- "file.timeout", JSON.Int 2000_L;
- "file.readahead", JSON.Int (1024_L *^ 1024_L);
- (* "file.sslverify", JSON.String "off"; XXX *)
- ] in
-
- (* Turn the JSON parameters into a 'json:' protocol string. *)
- "json: " ^ JSON.string_of_doc json_params
-
let parse_libvirt_xml ?conn xml debug "libvirt xml is:\n%s" xml;
@@ -334,7 +309,18 @@ let parse_libvirt_xml ?conn xml * without
needing VMware around.
*)
let path = Option.default "" (xpath_string
"source/@name") in
- let qemu_uri = create_curl_qemu_uri driver host port path in
+ let url + let port + match driver, port
with
+ | _, None -> ""
+ | "https", Some 443 -> ""
+ | "http", Some 80 -> ""
+ | _, Some port when port >= 1 -> ":" ^
string_of_int port
+ | _, Some port ->
+ invalid_arg "invalid port number in libvirt XML" in
+ sprintf "%s://%s%s%s" driver host port (uri_quote path)
in
+ let nbdkit = Nbdkit.create_curl ~password:NoPassword url in
+ let qemu_uri = Nbdkit.run nbdkit in
add_disk qemu_uri format controller P_dont_rewrite
| Some protocol, _, _ ->
warning (f_"<disk type='network'> with <source
protocol='%s'> was ignored")
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 09/12] v2v: vCenter: Replace qemu block curl driver with nbdkit-curl-plugin.
Because of the self-configuring readahead plugin we can entirely get
rid of input#adjust_overlay_parameters, which is definitely a good
thing.
---
v2v/Makefile.am | 1 +
v2v/input_libvirt_other.mli | 1 -
v2v/input_libvirt_vcenter_https.ml | 47 +++---------------------------
v2v/types.ml | 1 -
v2v/types.mli | 7 -----
v2v/v2v.ml | 7 -----
v2v/vCenter.ml | 41 ++++++--------------------
v2v/vCenter.mli | 4 +--
8 files changed, 16 insertions(+), 93 deletions(-)
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 2aa4b675e..55b966efa 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -284,6 +284,7 @@ COPY_TO_LOCAL_BOBJECTS = \
uefi.cmo \
utils.cmo \
libvirt_utils.cmo \
+ nbdkit.cmo \
vCenter.cmo \
copy_to_local.cmo
COPY_TO_LOCAL_XOBJECTS = $(COPY_TO_LOCAL_BOBJECTS:.cmo=.cmx)
diff --git a/v2v/input_libvirt_other.mli b/v2v/input_libvirt_other.mli
index 10f574281..d48987e46 100644
--- a/v2v/input_libvirt_other.mli
+++ b/v2v/input_libvirt_other.mli
@@ -24,7 +24,6 @@ class virtual input_libvirt : Libvirt.rw Libvirt.Connect.t
Lazy.t -> string -> o
method precheck : unit -> unit
method as_options : string
method virtual source : unit -> Types.source
- method adjust_overlay_parameters : Types.overlay -> unit
method private conn : Libvirt.rw Libvirt.Connect.t
end
diff --git a/v2v/input_libvirt_vcenter_https.ml
b/v2v/input_libvirt_vcenter_https.ml
index c7a00210e..bfe5e8e9d 100644
--- a/v2v/input_libvirt_vcenter_https.ml
+++ b/v2v/input_libvirt_vcenter_https.ml
@@ -30,17 +30,12 @@ open Input_libvirt_other
open Printf
-(* See RHBZ#1151033 and RHBZ#1153589. *)
-let readahead_for_conversion = None
-let readahead_for_copying = Some (64 * 1024 * 1024)
-
(* Subclass specialized for handling VMware vCenter over https. *)
class input_libvirt_vcenter_https
libvirt_conn input_password parsed_uri server guest object (self)
inherit input_libvirt libvirt_conn guest
- val saved_source_paths = Hashtbl.create 13
val mutable dcPath = ""
method precheck () @@ -50,9 +45,9 @@ object (self)
debug "input_libvirt_vcenter_https: source: server %s" server;
(* Remove proxy environment variables so curl doesn't try to use
- * them. Libvirt doesn't use the proxy anyway, and using a proxy
- * is generally a bad idea because vCenter is slow enough as it is
- * without putting another device in the way (RHBZ#1354507).
+ * them. Using a proxy is generally a bad idea because vCenter
+ * is slow enough as it is without putting another device in
+ * the way (RHBZ#1354507).
*)
unsetenv "https_proxy";
unsetenv "all_proxy";
@@ -77,28 +72,13 @@ object (self)
error (f_"vcenter: <vmware:datacenterpath> was not found in
the XML. You need to upgrade to libvirt ≥ 1.2.20.")
);
- (* Save the original source paths, so that we can remap them again
- * in [#adjust_overlay_parameters].
- *)
- List.iter (
- function
- | { p_source = P_source_dev _ } ->
- (* Should never happen ... *)
- error (f_"source disk has <source dev=...> attribute in
XML")
- | { p_source_disk = { s_disk_id = id }; p_source = P_dont_rewrite } ->
- Hashtbl.add saved_source_paths id None
- | { p_source_disk = { s_disk_id = id }; p_source = P_source_file path }
->
- Hashtbl.add saved_source_paths id (Some path)
- ) disks;
-
- let readahead = readahead_for_conversion in
let disks = List.map (
function
| { p_source = P_source_dev _ } -> assert false
| { p_source_disk = disk; p_source = P_dont_rewrite } -> disk
| { p_source_disk = disk; p_source = P_source_file path } ->
let { VCenter.qemu_uri } - VCenter.map_source ?readahead
?password_file:input_password
+ VCenter.map_source ?password_file:input_password
dcPath parsed_uri server path in
(* The libvirt ESX driver doesn't normally specify a format, but
@@ -108,25 +88,6 @@ object (self)
) disks in
{ source with s_disks = disks }
-
- (* See RHBZ#1151033 and RHBZ#1153589 for why this is necessary. *)
- method adjust_overlay_parameters overlay - let orig_path - try
Hashtbl.find saved_source_paths overlay.ov_source.s_disk_id
- with Not_found -> failwith "internal error in
adjust_overlay_parameters" in
- match orig_path with
- | None -> ()
- | Some orig_path ->
- let readahead = readahead_for_copying in
- let { VCenter.qemu_uri = backing_qemu_uri } - VCenter.map_source
?readahead ?password_file:input_password
- dcPath parsed_uri server orig_path in
-
- (* Rebase the qcow2 overlay to adjust the readahead parameter. *)
- let cmd = [ "qemu-img"; "rebase"; "-u";
"-b"; backing_qemu_uri;
- overlay.ov_overlay_file ] in
- if run_command cmd <> 0 then
- warning (f_"qemu-img rebase failed (ignored)")
end
let input_libvirt_vcenter_https = new input_libvirt_vcenter_https
diff --git a/v2v/types.ml b/v2v/types.ml
index 714b30014..1100dd2a6 100644
--- a/v2v/types.ml
+++ b/v2v/types.ml
@@ -510,7 +510,6 @@ class virtual input = object
method precheck () = ()
method virtual as_options : string
method virtual source : unit -> source
- method adjust_overlay_parameters (_ : overlay) = ()
end
class virtual output = object
diff --git a/v2v/types.mli b/v2v/types.mli
index f595ab0ef..d605b1ad4 100644
--- a/v2v/types.mli
+++ b/v2v/types.mli
@@ -384,10 +384,6 @@ type output_allocation = Sparse | Preallocated
│
│
▼
- input#adjust_overlay_parameters Optional method for adjusting
- │ QEMU overlay parameters ready for copying
- │ (eg. using a larger readahead setting).
- ▼
copying Guest data is copied to the target disks
by running ‘qemu-img convert’.
v}
@@ -404,9 +400,6 @@ class virtual input : object
This is just used for pretty-printing log messages. *)
method virtual source : unit -> source
(** Examine the source hypervisor and create a source struct. *)
- method adjust_overlay_parameters : overlay -> unit
- (** Called just before copying to allow the input module to adjust
- parameters of the overlay disk. *)
end
(** Encapsulates all [-i], etc input arguments as an object. *)
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 4ee15663f..afe46cd9b 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -700,13 +700,6 @@ and copy_targets cmdline targets input output if not
((open_guestfs ())#disk_has_backing_file overlay_file) then
error (f_"internal error: qemu corrupted the overlay file");
- (* Give the input module a chance to adjust the parameters
- * of the overlay/backing file. This allows us to increase
- * the readahead parameter when copying (see RHBZ#1151033 and
- * RHBZ#1153589 for the gruesome details).
- *)
- input#adjust_overlay_parameters t.target_overlay;
-
(match t.target_file with
| TargetFile filename ->
(* It turns out that libguestfs's disk creation code is
diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml
index b1b9f9b15..2563ad0ed 100644
--- a/v2v/vCenter.ml
+++ b/v2v/vCenter.ml
@@ -35,7 +35,7 @@ type remote_resource = {
let source_re = PCRE.compile "^\\[(.*)\\] (.*)\\.vmdk$"
let snapshot_re = PCRE.compile "^(.*)-\\d{6}(\\.vmdk)$"
-let rec map_source ?readahead ?password_file dcPath uri server path +let rec
map_source ?password_file dcPath uri server path (* If no_verify=1 was passed
in the libvirt URI, then we have to
* turn off certificate verification here too.
*)
@@ -72,38 +72,15 @@ let rec map_source ?readahead ?password_file dcPath uri
server path let session_cookie get_session_cookie password_file uri
sslverify https_url in
- let qemu_uri - (* Construct the JSON parameters for the qemu URI. *)
- let json_params = [
- "file.driver", JSON.String "https";
- "file.url", JSON.String https_url;
- (* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *)
- "file.timeout", JSON.Int 2000_L;
- ] in
-
- let json_params - match readahead with
- | None -> json_params
- | Some readahead ->
- ("file.readahead", JSON.Int (Int64.of_int readahead)) ::
json_params in
-
- let json_params - if sslverify then json_params
- else ("file.sslverify", JSON.String "off") ::
json_params in
-
- let json_params - match session_cookie with
- | None -> json_params
- | Some cookie -> ("file.cookie", JSON.String cookie) ::
json_params in
-
- debug "vcenter: json parameters: %s" (JSON.string_of_doc
json_params);
-
- (* Turn the JSON parameters into a 'json:' protocol string.
- * Note this requires qemu-img >= 2.1.0.
- *)
- let qemu_uri = "json: " ^ JSON.string_of_doc json_params in
+ let password + match password_file with
+ | None -> Nbdkit.NoPassword
+ | Some password_file -> Nbdkit.PasswordFile password_file in
- qemu_uri in
+ let nbdkit + Nbdkit.create_curl ?cookie:session_cookie ~password
~sslverify
+ https_url in
+ let qemu_uri = Nbdkit.run nbdkit in
(* Return the struct. *)
{ https_url = https_url;
diff --git a/v2v/vCenter.mli b/v2v/vCenter.mli
index 840e0a09e..d72d5686e 100644
--- a/v2v/vCenter.mli
+++ b/v2v/vCenter.mli
@@ -54,8 +54,8 @@ type remote_resource = {
(** The "remote resource" is the structure returned by the
{!map_source}
function. *)
-val map_source : ?readahead:int -> ?password_file:string -> string ->
Xml.uri -> string -> string -> remote_resource
-(** [map_source ?readahead ?password_file dcPath uri server path]
+val map_source : ?password_file:string -> string -> Xml.uri -> string
-> string -> remote_resource
+(** [map_source ?password_file dcPath uri server path]
maps the [<source path=...>] string to a {!remote_resource}
structure containing both an [https://] URL and a qemu URI,
both pointing the guest disk.
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 10/12] v2v: Implement the --bandwidth* options to control network bandwidth.
For input methods which use nbdkit, we can cheaply add
nbdkit-rate-filter to control input-side network bandwidth. These
options control that filter. We can choose to set the bandwidth
statically and optionally change it dynamically:
--bandwidth 10M
# static bandwidth of 10 Mbps, no dynamic adjustment possible
--bandwidth 5M --bandwidth-file /tmp/bw
# initial static bandwidth of 5 Mbps, adjustable by writing to /tmp/bw
--bandwidth-file /tmp/bw
# no initial bandwidth cap, can be added later by writing to /tmp/bw
It only makes sense to control the input side since virt-v2v writes a
lot less data than it reads.
---
v2v/Makefile.am | 1 +
v2v/cmdline.ml | 17 ++++++++++--
v2v/cmdline.mli | 1 +
v2v/input_disk.ml | 2 +-
v2v/input_libvirt_other.ml | 4 +--
v2v/input_libvirt_other.mli | 2 +-
v2v/input_libvirt_vcenter_https.ml | 6 ++--
v2v/input_libvirt_vddk.ml | 6 ++--
v2v/input_libvirt_xen_ssh.ml | 6 ++--
v2v/input_libvirtxml.ml | 2 +-
v2v/input_ova.ml | 2 +-
v2v/input_vmx.ml | 26 ++++++++++--------
v2v/nbdkit.ml | 34 +++++++++++++++++------
v2v/nbdkit.mli | 9 ++++--
v2v/parse_libvirt_xml.ml | 9 +++---
v2v/parse_libvirt_xml.mli | 4 +--
v2v/types.ml | 6 +++-
v2v/types.mli | 7 ++++-
v2v/v2v.ml | 3 +-
v2v/vCenter.ml | 4 +--
v2v/vCenter.mli | 3 +-
v2v/virt-v2v.pod | 44 ++++++++++++++++++++++++++++++
22 files changed, 146 insertions(+), 52 deletions(-)
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 55b966efa..63d9b963a 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -281,6 +281,7 @@ virt_v2v_copy_to_local_CFLAGS = \
$(LIBVIRT_CFLAGS)
COPY_TO_LOCAL_BOBJECTS = \
+ types.cmo \
uefi.cmo \
utils.cmo \
libvirt_utils.cmo \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 4d390f249..641eed017 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -29,6 +29,7 @@ open Types
open Utils
type cmdline = {
+ bandwidth : bandwidth option;
compressed : bool;
debug_overlays : bool;
do_copy : bool;
@@ -47,6 +48,8 @@ type cmdline = {
let mac_re = PCRE.compile ~anchored:true
"([[:xdigit:]]{2}:[[:xdigit:]]{2}:[[:xdigit:]]{2}:[[:xdigit:]]{2}:[[:xdigit:]]{2}:[[:xdigit:]]{2}):(network|bridge):(.*)"
let parse_cmdline () + let bandwidth = ref None in
+ let bandwidth_file = ref None in
let compressed = ref false in
let debug_overlays = ref false in
let do_copy = ref true in
@@ -191,6 +194,10 @@ let parse_cmdline () and ovf_flavours_str =
String.concat "|" Create_ovf.ovf_flavours in
let argspec = [
+ [ L"bandwidth" ], Getopt.String ("bps",
set_string_option_once "--bandwidth" bandwidth),
+ s_"Set bandwidth to bits per
sec";
+ [ L"bandwidth-file" ], Getopt.String ("filename",
set_string_option_once "--bandwidth-file" bandwidth_file),
+ s_"Set bandwidth dynamically from
file";
[ S 'b'; L"bridge" ], Getopt.String ("in:out",
add_bridge),
s_"Map bridge ‘in’ to ‘out’";
[ L"compressed" ], Getopt.Set compressed,
@@ -304,6 +311,11 @@ read the man page virt-v2v(1).
(* Dereference the arguments. *)
let args = List.rev !args in
+ let bandwidth + match !bandwidth, !bandwidth_file with
+ | None, None -> None
+ | Some rate, None -> Some (StaticBandwidth rate)
+ | rate, Some filename -> Some (DynamicBandwidth (rate, filename)) in
let compressed = !compressed in
let debug_overlays = !debug_overlays in
let do_copy = !do_copy in
@@ -351,6 +363,7 @@ read the man page virt-v2v(1).
pr "in-place\n";
pr "io/oo\n";
pr "mac-option\n";
+ pr "bandwidth-option\n";
List.iter (pr "input:%s\n") (Modules_list.input_modules ());
List.iter (pr "output:%s\n") (Modules_list.output_modules ());
List.iter (pr "convert:%s\n") (Modules_list.convert_modules ());
@@ -683,8 +696,8 @@ read the man page virt-v2v(1).
output_format, output_alloc in
{
- compressed; debug_overlays; do_copy; in_place; network_map;
- output_alloc; output_format; output_name;
+ bandwidth; compressed; debug_overlays; do_copy; in_place;
+ network_map; output_alloc; output_format; output_name;
print_estimate; print_source; root_choice;
ks = opthandle.ks;
},
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 78601e191..1c9e6c258 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -19,6 +19,7 @@
(** Command line argument parsing. *)
type cmdline = {
+ bandwidth : Types.bandwidth option;
compressed : bool;
debug_overlays : bool;
do_copy : bool;
diff --git a/v2v/input_disk.ml b/v2v/input_disk.ml
index 8321a2a8c..52f40a31b 100644
--- a/v2v/input_disk.ml
+++ b/v2v/input_disk.ml
@@ -36,7 +36,7 @@ class input_disk input_format disk = object
| Some fmt -> " -if " ^ fmt)
disk
- method source () + method source ?bandwidth () (* Check the input file
exists and is readable. *)
Unix.access disk [Unix.R_OK];
diff --git a/v2v/input_libvirt_other.ml b/v2v/input_libvirt_other.ml
index 5ff3cfc43..504c72600 100644
--- a/v2v/input_libvirt_other.ml
+++ b/v2v/input_libvirt_other.ml
@@ -58,10 +58,10 @@ class input_libvirt_other libvirt_conn guest object (self)
inherit input_libvirt libvirt_conn guest
- method source () + method source ?bandwidth () debug
"input_libvirt_other: source ()";
- let source, disks, _ = parse_libvirt_domain self#conn guest in
+ let source, disks, _ = parse_libvirt_domain ?bandwidth self#conn guest in
let disks = List.map (fun { p_source_disk = disk } -> disk) disks in
{ source with s_disks = disks }
end
diff --git a/v2v/input_libvirt_other.mli b/v2v/input_libvirt_other.mli
index d48987e46..1aac92ad6 100644
--- a/v2v/input_libvirt_other.mli
+++ b/v2v/input_libvirt_other.mli
@@ -23,7 +23,7 @@ val error_if_libvirt_does_not_support_json_backingfile : unit
-> unit
class virtual input_libvirt : Libvirt.rw Libvirt.Connect.t Lazy.t -> string
-> object
method precheck : unit -> unit
method as_options : string
- method virtual source : unit -> Types.source
+ method virtual source : ?bandwidth:Types.bandwidth -> unit ->
Types.source
method private conn : Libvirt.rw Libvirt.Connect.t
end
diff --git a/v2v/input_libvirt_vcenter_https.ml
b/v2v/input_libvirt_vcenter_https.ml
index bfe5e8e9d..280febf98 100644
--- a/v2v/input_libvirt_vcenter_https.ml
+++ b/v2v/input_libvirt_vcenter_https.ml
@@ -41,7 +41,7 @@ object (self)
method precheck () error_if_libvirt_does_not_support_json_backingfile ()
- method source () + method source ?bandwidth () debug
"input_libvirt_vcenter_https: source: server %s" server;
(* Remove proxy environment variables so curl doesn't try to use
@@ -56,7 +56,7 @@ object (self)
unsetenv "ALL_PROXY";
unsetenv "NO_PROXY";
- let source, disks, xml = parse_libvirt_domain self#conn guest in
+ let source, disks, xml = parse_libvirt_domain ?bandwidth self#conn guest in
(* Find the <vmware:datacenterpath> element from the XML. This
* was added in libvirt >= 1.2.20.
@@ -78,7 +78,7 @@ object (self)
| { p_source_disk = disk; p_source = P_dont_rewrite } -> disk
| { p_source_disk = disk; p_source = P_source_file path } ->
let { VCenter.qemu_uri } - VCenter.map_source
?password_file:input_password
+ VCenter.map_source ?bandwidth ?password_file:input_password
dcPath parsed_uri server path in
(* The libvirt ESX driver doesn't normally specify a format, but
diff --git a/v2v/input_libvirt_vddk.ml b/v2v/input_libvirt_vddk.ml
index 1f54ee511..19b4166c7 100644
--- a/v2v/input_libvirt_vddk.ml
+++ b/v2v/input_libvirt_vddk.ml
@@ -113,8 +113,8 @@ object (self)
super#as_options (* superclass prints "-i libvirt etc" *)
pt_options
- method source () - let source, disks, xml = parse_libvirt_domain self#conn
guest in
+ method source ?bandwidth () + let source, disks, xml =
parse_libvirt_domain ?bandwidth self#conn guest in
(* Find the <vmware:moref> element from the XML. This was added
* in libvirt >= 3.7 and is required.
@@ -183,7 +183,7 @@ object (self)
* directly in this form to VDDK.
*)
let nbdkit - Nbdkit.create_vddk ?config ?cookie ?libdir
~moref
+ Nbdkit.create_vddk ?bandwidth ?config ?cookie ?libdir ~moref
?nfchostport ?password_file:input_password ?port
~server ?snapshot ~thumbprint ?transports ?user
path in
diff --git a/v2v/input_libvirt_xen_ssh.ml b/v2v/input_libvirt_xen_ssh.ml
index 9a941d070..f5877b054 100644
--- a/v2v/input_libvirt_xen_ssh.ml
+++ b/v2v/input_libvirt_xen_ssh.ml
@@ -40,10 +40,10 @@ object (self)
error_if_libvirt_does_not_support_json_backingfile ();
error_if_no_ssh_agent ()
- method source () + method source ?bandwidth () debug
"input_libvirt_xen_ssh: source: server %s" server;
- let source, disks, _ = parse_libvirt_domain self#conn guest in
+ let source, disks, _ = parse_libvirt_domain ?bandwidth self#conn guest in
let port match parsed_uri.uri_port with
@@ -61,7 +61,7 @@ object (self)
disk
| { p_source_disk = disk; p_source = P_source_dev path }
| { p_source_disk = disk; p_source = P_source_file path } ->
- let nbdkit = Nbdkit.create_ssh ~password:NoPassword
+ let nbdkit = Nbdkit.create_ssh ?bandwidth ~password:NoPassword
?port ~server ?user path in
let qemu_uri = Nbdkit.run nbdkit in
{ disk with s_qemu_uri = qemu_uri }
diff --git a/v2v/input_libvirtxml.ml b/v2v/input_libvirtxml.ml
index a44b41fce..efffb28b0 100644
--- a/v2v/input_libvirtxml.ml
+++ b/v2v/input_libvirtxml.ml
@@ -31,7 +31,7 @@ object
method as_options = "-i libvirtxml " ^ file
- method source () + method source ?bandwidth () let xml =
read_whole_file file in
let source, disks = parse_libvirt_xml xml in
diff --git a/v2v/input_ova.ml b/v2v/input_ova.ml
index 6309ff9a5..872796137 100644
--- a/v2v/input_ova.ml
+++ b/v2v/input_ova.ml
@@ -75,7 +75,7 @@ class input_ova ova = object
method as_options = "-i ova " ^ ova
- method source () + method source ?bandwidth () (* Extract ova file. *)
let ova_t = parse_ova ova in
diff --git a/v2v/input_vmx.ml b/v2v/input_vmx.ml
index 0db1945ea..1a7b331ee 100644
--- a/v2v/input_vmx.ml
+++ b/v2v/input_vmx.ml
@@ -112,8 +112,9 @@ let remote_file_exists uri path eprintf
"%s\n%!" cmd;
Sys.command cmd = 0
-let rec find_disks vmx vmx_source - find_scsi_disks vmx vmx_source @
find_ide_disks vmx vmx_source
+let rec find_disks ?bandwidth vmx vmx_source + find_scsi_disks ?bandwidth vmx
vmx_source
+ @ find_ide_disks ?bandwidth vmx vmx_source
(* Find all SCSI hard disks.
*
@@ -123,7 +124,7 @@ let rec find_disks vmx vmx_source *
| omitted
* scsi0:0.fileName = "guest.vmdk"
*)
-and find_scsi_disks vmx vmx_source +and find_scsi_disks ?bandwidth vmx
vmx_source let get_scsi_controller_target ns sscanf ns
"scsi%d:%d" (fun c t -> c, t)
in
@@ -135,7 +136,7 @@ and find_scsi_disks vmx vmx_source
Some "scsi-harddisk"; None ] in
let scsi_controller = Source_SCSI in
- find_hdds vmx vmx_source
+ find_hdds ?bandwidth vmx vmx_source
get_scsi_controller_target is_scsi_controller_target
scsi_device_types scsi_controller
@@ -145,7 +146,7 @@ and find_scsi_disks vmx vmx_source * ide0:0.deviceType =
"ata-hardDisk"
* ide0:0.fileName = "guest.vmdk"
*)
-and find_ide_disks vmx vmx_source +and find_ide_disks ?bandwidth vmx vmx_source
let get_ide_controller_target ns sscanf ns "ide%d:%d" (fun c t
-> c, t)
in
@@ -156,11 +157,11 @@ and find_ide_disks vmx vmx_source let ide_device_types
= [ Some "ata-harddisk" ] in
let ide_controller = Source_IDE in
- find_hdds vmx vmx_source
+ find_hdds ?bandwidth vmx vmx_source
get_ide_controller_target is_ide_controller_target
ide_device_types ide_controller
-and find_hdds vmx vmx_source
+and find_hdds ?bandwidth vmx vmx_source
get_controller_target is_controller_target
device_types controller (* Find namespaces matching
'(ide|scsi)X:Y' with suitable deviceType. *)
@@ -186,7 +187,8 @@ and find_hdds vmx vmx_source
match path, v with
| [ns; "filename"], Some filename ->
let c, t = get_controller_target ns in
- let uri, format = qemu_uri_of_filename vmx_source filename in
+ let uri, format = qemu_uri_of_filename ?bandwidth
+ vmx_source filename in
let s = { s_disk_id = (-1);
s_qemu_uri = uri; s_format = Some format;
s_controller = Some controller } in
@@ -213,7 +215,7 @@ and find_hdds vmx vmx_source
* This constructs a QEMU URI of the filename relative to the
* vmx file (which might be remote over SSH).
*)
-and qemu_uri_of_filename vmx_source filename +and qemu_uri_of_filename
?bandwidth vmx_source filename match vmx_source with
| File vmx_filename ->
(* Always ensure this returns an absolute path to avoid
@@ -240,7 +242,7 @@ and qemu_uri_of_filename vmx_source filename let port
= Option.map string_of_int (port_of_uri uri) in
let user = uri.Xml.uri_user in
- let nbdkit = Nbdkit.create_ssh ~password:NoPassword ~server
+ let nbdkit = Nbdkit.create_ssh ?bandwidth ~password:NoPassword ~server
?port ?user abs_path in
let qemu_uri = Nbdkit.run nbdkit in
qemu_uri, format
@@ -392,7 +394,7 @@ object
method as_options = "-i vmx " ^ arg
- method source () + method source ?bandwidth () let vmx_source =
vmx_source_of_arg input_transport arg in
(* If the transport is SSH, fetch the file from remote, else
@@ -486,7 +488,7 @@ object
None
| None -> None in
- let disks = find_disks vmx vmx_source in
+ let disks = find_disks ?bandwidth vmx vmx_source in
let removables = find_removables vmx in
let nics = find_nics vmx in
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index b2d77f963..776eedce0 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -24,6 +24,7 @@ open Std_utils
open Tools_utils
open Unix_utils
+open Types
open Utils
let nbdkit_min_version = (1, 12)
@@ -88,7 +89,7 @@ let error_unless_nbdkit_compiled_with_selinux dump_config
error (f_"nbdkit was compiled without SELinux support. You will have to
recompile nbdkit with libselinux-devel installed, or else set SELinux to
Permissive mode while doing the conversion.")
)
-let common_create plugin_name plugin_args plugin_env +let common_create
?bandwidth plugin_name plugin_args plugin_env error_unless_nbdkit_working ();
(* Environment. We always add LANG=C. *)
@@ -150,7 +151,24 @@ let common_create plugin_name plugin_args plugin_env
add_arg "--filter"; add_arg "readahead"
);
- let args = get_args () @ [ plugin_name ] @ plugin_args in
+ (* Add the rate filter. *)
+ let rate_args + if Sys.file_exists (filterdir //
"nbdkit-rate-filter.so") then (
+ match bandwidth with
+ | None -> []
+ | Some bandwidth ->
+ add_arg "--filter"; add_arg "rate";
+ match bandwidth with
+ | StaticBandwidth rate ->
+ [ "rate=" ^ rate ]
+ | DynamicBandwidth (None, filename) ->
+ [ "rate-file=" ^ filename ]
+ | DynamicBandwidth (Some rate, filename) ->
+ [ "rate=" ^ rate; "rate-file=" ^ filename ]
+ )
+ else [] in
+
+ let args = get_args () @ [ plugin_name ] @ plugin_args @ rate_args in
{ plugin_name; args; env; dump_config; dump_plugin; filterdir }
@@ -160,7 +178,7 @@ let common_create plugin_name plugin_args plugin_env let
libNN = sprintf "lib%d" Sys.word_size
(* Create an nbdkit module specialized for reading from VDDK sources. *)
-let create_vddk ?config ?cookie ?libdir ~moref
+let create_vddk ?bandwidth ?config ?cookie ?libdir ~moref
?nfchostport ?password_file ?port
~server ?snapshot ~thumbprint ?transports ?user path (*
Compute the LD_LIBRARY_PATH that we may have to pass to nbdkit. *)
@@ -255,10 +273,10 @@ See also the virt-v2v-input-vmware(1) manual.") libNN
add_arg (sprintf "thumbprint=%s" thumbprint);
Option.may (fun s -> add_arg (sprintf "transports=%s" s))
transports;
- common_create "vddk" (get_args ()) env
+ common_create ?bandwidth "vddk" (get_args ()) env
(* Create an nbdkit module specialized for reading from SSH sources. *)
-let create_ssh ~password ?port ~server ?user path +let create_ssh ?bandwidth
~password ?port ~server ?user path let add_arg, get_args let args = ref
[] in
let add_arg a = List.push_front a args in
@@ -276,10 +294,10 @@ let create_ssh ~password ?port ~server ?user path );
add_arg (sprintf "path=%s" path);
- common_create "ssh" (get_args ()) []
+ common_create ?bandwidth "ssh" (get_args ()) []
(* Create an nbdkit module specialized for reading from Curl sources. *)
-let create_curl ?cookie ~password ?(sslverify=true) ?user url +let create_curl
?bandwidth ?cookie ~password ?(sslverify=true) ?user url let add_arg,
get_args let args = ref [] in
let add_arg a = List.push_front a args in
@@ -299,7 +317,7 @@ let create_curl ?cookie ~password ?(sslverify=true) ?user
url if not sslverify then add_arg "sslverify=false";
add_arg (sprintf "url=%s" url);
- common_create "curl" (get_args ()) []
+ common_create ?bandwidth "curl" (get_args ()) []
let run { args; env } (* Create a temporary directory where we place the
sockets. *)
diff --git a/v2v/nbdkit.mli b/v2v/nbdkit.mli
index efbb62f98..627c78c11 100644
--- a/v2v/nbdkit.mli
+++ b/v2v/nbdkit.mli
@@ -20,7 +20,8 @@
type t
-val create_vddk : ?config:string ->
+val create_vddk : ?bandwidth:Types.bandwidth ->
+ ?config:string ->
?cookie:string ->
?libdir:string ->
moref:string ->
@@ -46,7 +47,8 @@ type password | AskForPassword
| PasswordFile of string
-val create_ssh : password:password ->
+val create_ssh : ?bandwidth:Types.bandwidth ->
+ password:password ->
?port:string ->
server:string ->
?user:string ->
@@ -59,7 +61,8 @@ val create_ssh : password:password ->
Note this doesn't run nbdkit yet, it just creates the object. *)
-val create_curl : ?cookie:string ->
+val create_curl : ?bandwidth:Types.bandwidth ->
+ ?cookie:string ->
password:password ->
?sslverify:bool ->
?user:string ->
diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml
index 97d8a5cd8..86990aeb3 100644
--- a/v2v/parse_libvirt_xml.ml
+++ b/v2v/parse_libvirt_xml.ml
@@ -46,7 +46,7 @@ let get_drive_slot str offset warning (f_"could
not parse device name ‘%s’ from the source libvirt XML") str;
None
-let parse_libvirt_xml ?conn xml +let parse_libvirt_xml ?bandwidth ?conn xml
debug "libvirt xml is:\n%s" xml;
(* Create a default libvirt connection on request, to not open one
@@ -319,7 +319,8 @@ let parse_libvirt_xml ?conn xml | _, Some
port ->
invalid_arg "invalid port number in libvirt XML" in
sprintf "%s://%s%s%s" driver host port (uri_quote path)
in
- let nbdkit = Nbdkit.create_curl ~password:NoPassword url in
+ let nbdkit = Nbdkit.create_curl ?bandwidth ~password:NoPassword
+ url in
let qemu_uri = Nbdkit.run nbdkit in
add_disk qemu_uri format controller P_dont_rewrite
| Some protocol, _, _ ->
@@ -537,9 +538,9 @@ let parse_libvirt_xml ?conn xml },
disks)
-let parse_libvirt_domain conn guest +let parse_libvirt_domain ?bandwidth conn
guest let dom = Libvirt_utils.get_domain conn guest in
(* Use XmlSecure to get passwords (RHBZ#1174123). *)
let xml = Libvirt.Domain.get_xml_desc_flags dom [Libvirt.Domain.XmlSecure] in
- let source, disks = parse_libvirt_xml ~conn xml in
+ let source, disks = parse_libvirt_xml ?bandwidth ~conn xml in
source, disks, xml
diff --git a/v2v/parse_libvirt_xml.mli b/v2v/parse_libvirt_xml.mli
index 2d81e0d99..658ebc5eb 100644
--- a/v2v/parse_libvirt_xml.mli
+++ b/v2v/parse_libvirt_xml.mli
@@ -27,7 +27,7 @@ and parsed_source | P_source_file of string (**
<source file> *)
| P_dont_rewrite (** s_qemu_uri is already set. *)
-val parse_libvirt_domain : Libvirt.rw Libvirt.Connect.t -> string ->
Types.source * parsed_disk list * string
+val parse_libvirt_domain : ?bandwidth:Types.bandwidth -> Libvirt.rw
Libvirt.Connect.t -> string -> Types.source * parsed_disk list * string
(** [parse_libvirt_domain conn dom] loads the XML of the domain [dom]
from the libvirt connection [conn].
The result is a tuple with a {!Types.source} structure, a list of
@@ -36,7 +36,7 @@ val parse_libvirt_domain : Libvirt.rw Libvirt.Connect.t ->
string -> Types.sourc
{b Note} the [source.s_disks] field is an empty list. The caller
must map over the parsed disks and update the [source.s_disks] field. *)
-val parse_libvirt_xml : ?conn:Libvirt.rw Libvirt.Connect.t -> string ->
Types.source * parsed_disk list
+val parse_libvirt_xml : ?bandwidth:Types.bandwidth -> ?conn:Libvirt.rw
Libvirt.Connect.t -> string -> Types.source * parsed_disk list
(** Take libvirt XML and parse it into a {!Types.source} structure and a
list of source disks.
diff --git a/v2v/types.ml b/v2v/types.ml
index 1100dd2a6..d406caeb9 100644
--- a/v2v/types.ml
+++ b/v2v/types.ml
@@ -506,10 +506,14 @@ type root_choice = AskRoot | SingleRoot | FirstRoot |
RootDev of string
type output_allocation = Sparse | Preallocated
+type bandwidth +| StaticBandwidth of string
+| DynamicBandwidth of string option * string
+
class virtual input = object
method precheck () = ()
method virtual as_options : string
- method virtual source : unit -> source
+ method virtual source : ?bandwidth:bandwidth -> unit -> source
end
class virtual output = object
diff --git a/v2v/types.mli b/v2v/types.mli
index d605b1ad4..556f6930f 100644
--- a/v2v/types.mli
+++ b/v2v/types.mli
@@ -361,6 +361,11 @@ type root_choice = AskRoot | SingleRoot | FirstRoot |
RootDev of string
type output_allocation = Sparse | Preallocated
(** Type of [-oa] (output allocation) option. *)
+type bandwidth +| StaticBandwidth of string
+| DynamicBandwidth of string option * string
+(** [--bandwidth] and [--bandwidth-file] options. *)
+
(** {2 Input object}
This is subclassed for the various input [-i] options.
@@ -398,7 +403,7 @@ class virtual input : object
method virtual as_options : string
(** Converts the input object back to the equivalent command line options.
This is just used for pretty-printing log messages. *)
- method virtual source : unit -> source
+ method virtual source : ?bandwidth:bandwidth -> unit -> source
(** Examine the source hypervisor and create a source struct. *)
end
(** Encapsulates all [-i], etc input arguments as an object. *)
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index afe46cd9b..e4b4dfe37 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -198,7 +198,8 @@ let rec main ()
and open_source cmdline input message (f_"Opening the source %s")
input#as_options;
- let source = input#source () in
+ let bandwidth = cmdline.bandwidth in
+ let source = input#source ?bandwidth () in
(* Print source and stop. *)
if cmdline.print_source then (
diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml
index 2563ad0ed..89c5579b9 100644
--- a/v2v/vCenter.ml
+++ b/v2v/vCenter.ml
@@ -35,7 +35,7 @@ type remote_resource = {
let source_re = PCRE.compile "^\\[(.*)\\] (.*)\\.vmdk$"
let snapshot_re = PCRE.compile "^(.*)-\\d{6}(\\.vmdk)$"
-let rec map_source ?password_file dcPath uri server path +let rec map_source
?bandwidth ?password_file dcPath uri server path (* If no_verify=1 was passed
in the libvirt URI, then we have to
* turn off certificate verification here too.
*)
@@ -78,7 +78,7 @@ let rec map_source ?password_file dcPath uri server path
| Some password_file -> Nbdkit.PasswordFile password_file in
let nbdkit - Nbdkit.create_curl ?cookie:session_cookie ~password
~sslverify
+ Nbdkit.create_curl ?bandwidth ?cookie:session_cookie ~password ~sslverify
https_url in
let qemu_uri = Nbdkit.run nbdkit in
diff --git a/v2v/vCenter.mli b/v2v/vCenter.mli
index d72d5686e..5620cad45 100644
--- a/v2v/vCenter.mli
+++ b/v2v/vCenter.mli
@@ -54,7 +54,8 @@ type remote_resource = {
(** The "remote resource" is the structure returned by the
{!map_source}
function. *)
-val map_source : ?password_file:string -> string -> Xml.uri -> string
-> string -> remote_resource
+val map_source : ?bandwidth:Types.bandwidth -> ?password_file:string ->
+ string -> Xml.uri -> string -> string ->
remote_resource
(** [map_source ?password_file dcPath uri server path]
maps the [<source path=...>] string to a {!remote_resource}
structure containing both an [https://] URL and a qemu URI,
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index 9a555c3be..8ba141be9 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -155,6 +155,50 @@ qemu, do:
Display help.
+=item B<--bandwidth> bps
+
+=item B<--bandwidth-file> filename
+
+Some input methods are able to limit the network bandwidth they will
+use statically or dynamically. In the first variant this sets the
+bandwidth limit statically in bits per second. Formats like C<10M>
+may be used (meaning 10 megabits per second).
+
+In the second variant the bandwidth is limited dynamically from the
+content of the file (also in bits per second, in the same formats
+supported by the first variant). You may use both parameters
+together, meaning: first limit to a static rate, then you can create
+the file while virt-v2v is running to adjust the rate dynamically.
+
+This is only supported for:
+
+=over 4
+
+=item *
+
+L<input from Xen|virt-v2v-input-xen(1)>
+
+=item *
+
+L<input from VMware VMX|virt-v2v-input-vmware(1)/INPUT FROM VMWARE VMX>
+when using the SSH transport method
+
+=item *
+
+L<input from VDDK|virt-v2v-input-vmware(1)/INPUT FROM VDDK>
+
+=item *
+
+I<-i libvirtxml> when using HTTP or HTTPS disks
+
+=item *
+
+L<input from VMware vCenter server|virt-v2v-input-vmware(1)/INPUT FROM
VMWARE VCENTER SERVER>
+
+=back
+
+The options are silently ignored for other input methods.
+
=item B<-b> ...
=item B<--bridge> ...
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 11/12] v2v: Implement SSH password authentication for Xen and VMX over SSH.
For example:
$ virt-v2v -i vmx -it ssh -ip /tmp/passwd \
'ssh://root@esxi/vmfs/volumes/datastore1/Windows/Windows.vmx' -o
null
---
v2v/cmdline.ml | 2 +-
v2v/input_libvirt.ml | 2 +-
v2v/input_libvirt_xen_ssh.ml | 8 ++++++--
v2v/input_libvirt_xen_ssh.mli | 2 +-
v2v/input_vmx.ml | 30 +++++++++++++++++-------------
v2v/input_vmx.mli | 4 ++--
v2v/virt-v2v-input-vmware.pod | 19 ++++++++++---------
v2v/virt-v2v-input-xen.pod | 25 ++++++++++++-------------
8 files changed, 50 insertions(+), 42 deletions(-)
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 641eed017..c6d7af09d 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -533,7 +533,7 @@ read the man page virt-v2v(1).
| Some `SSH -> Some `SSH
| Some (`VDDK _) ->
error (f_"only ‘-it ssh’ can be used here") in
- Input_vmx.input_vmx input_transport arg in
+ Input_vmx.input_vmx input_password input_transport arg in
(* Common error message. *)
let error_option_cannot_be_used_in_output_mode mode opt diff --git
a/v2v/input_libvirt.ml b/v2v/input_libvirt.ml
index a7d1706de..c41c2f60d 100644
--- a/v2v/input_libvirt.ml
+++ b/v2v/input_libvirt.ml
@@ -68,7 +68,7 @@ let input_libvirt input_conn input_password input_transport
guest (* Xen over SSH *)
| Some server, Some "xen+ssh", _ ->
Input_libvirt_xen_ssh.input_libvirt_xen_ssh
- libvirt_conn parsed_uri server guest
+ libvirt_conn input_password parsed_uri server guest
(* Old virt-v2v also supported qemu+ssh://. However I am
* deliberately not supporting this in new virt-v2v. Don't
diff --git a/v2v/input_libvirt_xen_ssh.ml b/v2v/input_libvirt_xen_ssh.ml
index f5877b054..829406cbf 100644
--- a/v2v/input_libvirt_xen_ssh.ml
+++ b/v2v/input_libvirt_xen_ssh.ml
@@ -30,7 +30,7 @@ open Input_libvirt_other
open Printf
(* Subclass specialized for handling Xen over SSH. *)
-class input_libvirt_xen_ssh libvirt_conn parsed_uri server guest +class
input_libvirt_xen_ssh libvirt_conn input_password parsed_uri server guest
object (self)
inherit input_libvirt libvirt_conn guest
@@ -61,7 +61,11 @@ object (self)
disk
| { p_source_disk = disk; p_source = P_source_dev path }
| { p_source_disk = disk; p_source = P_source_file path } ->
- let nbdkit = Nbdkit.create_ssh ?bandwidth ~password:NoPassword
+ let password + match input_password with
+ | None -> Nbdkit.NoPassword
+ | Some ip -> Nbdkit.PasswordFile ip in
+ let nbdkit = Nbdkit.create_ssh ?bandwidth ~password
?port ~server ?user path in
let qemu_uri = Nbdkit.run nbdkit in
{ disk with s_qemu_uri = qemu_uri }
diff --git a/v2v/input_libvirt_xen_ssh.mli b/v2v/input_libvirt_xen_ssh.mli
index 6649b9883..037d1ffbf 100644
--- a/v2v/input_libvirt_xen_ssh.mli
+++ b/v2v/input_libvirt_xen_ssh.mli
@@ -18,4 +18,4 @@
(** [-i libvirt] when the source is Xen *)
-val input_libvirt_xen_ssh : Libvirt.rw Libvirt.Connect.t Lazy.t -> Xml.uri
-> string -> string -> Types.input
+val input_libvirt_xen_ssh : Libvirt.rw Libvirt.Connect.t Lazy.t -> string
option -> Xml.uri -> string -> string -> Types.input
diff --git a/v2v/input_vmx.ml b/v2v/input_vmx.ml
index 1a7b331ee..0b5050506 100644
--- a/v2v/input_vmx.ml
+++ b/v2v/input_vmx.ml
@@ -112,9 +112,9 @@ let remote_file_exists uri path eprintf
"%s\n%!" cmd;
Sys.command cmd = 0
-let rec find_disks ?bandwidth vmx vmx_source - find_scsi_disks ?bandwidth vmx
vmx_source
- @ find_ide_disks ?bandwidth vmx vmx_source
+let rec find_disks ?bandwidth input_password vmx vmx_source + find_scsi_disks
?bandwidth input_password vmx vmx_source
+ @ find_ide_disks ?bandwidth input_password vmx vmx_source
(* Find all SCSI hard disks.
*
@@ -124,7 +124,7 @@ let rec find_disks ?bandwidth vmx vmx_source *
| omitted
* scsi0:0.fileName = "guest.vmdk"
*)
-and find_scsi_disks ?bandwidth vmx vmx_source +and find_scsi_disks ?bandwidth
input_password vmx vmx_source let get_scsi_controller_target ns sscanf
ns "scsi%d:%d" (fun c t -> c, t)
in
@@ -136,7 +136,7 @@ and find_scsi_disks ?bandwidth vmx vmx_source
Some "scsi-harddisk"; None ] in
let scsi_controller = Source_SCSI in
- find_hdds ?bandwidth vmx vmx_source
+ find_hdds ?bandwidth input_password vmx vmx_source
get_scsi_controller_target is_scsi_controller_target
scsi_device_types scsi_controller
@@ -146,7 +146,7 @@ and find_scsi_disks ?bandwidth vmx vmx_source *
ide0:0.deviceType = "ata-hardDisk"
* ide0:0.fileName = "guest.vmdk"
*)
-and find_ide_disks ?bandwidth vmx vmx_source +and find_ide_disks ?bandwidth
input_password vmx vmx_source let get_ide_controller_target ns sscanf ns
"ide%d:%d" (fun c t -> c, t)
in
@@ -157,11 +157,11 @@ and find_ide_disks ?bandwidth vmx vmx_source let
ide_device_types = [ Some "ata-harddisk" ] in
let ide_controller = Source_IDE in
- find_hdds ?bandwidth vmx vmx_source
+ find_hdds ?bandwidth input_password vmx vmx_source
get_ide_controller_target is_ide_controller_target
ide_device_types ide_controller
-and find_hdds ?bandwidth vmx vmx_source
+and find_hdds ?bandwidth input_password vmx vmx_source
get_controller_target is_controller_target
device_types controller (* Find namespaces matching
'(ide|scsi)X:Y' with suitable deviceType. *)
@@ -187,7 +187,7 @@ and find_hdds ?bandwidth vmx vmx_source
match path, v with
| [ns; "filename"], Some filename ->
let c, t = get_controller_target ns in
- let uri, format = qemu_uri_of_filename ?bandwidth
+ let uri, format = qemu_uri_of_filename ?bandwidth input_password
vmx_source filename in
let s = { s_disk_id = (-1);
s_qemu_uri = uri; s_format = Some format;
@@ -215,7 +215,7 @@ and find_hdds ?bandwidth vmx vmx_source
* This constructs a QEMU URI of the filename relative to the
* vmx file (which might be remote over SSH).
*)
-and qemu_uri_of_filename ?bandwidth vmx_source filename +and
qemu_uri_of_filename ?bandwidth input_password vmx_source filename match
vmx_source with
| File vmx_filename ->
(* Always ensure this returns an absolute path to avoid
@@ -241,8 +241,12 @@ and qemu_uri_of_filename ?bandwidth vmx_source filename
let server = server_of_uri uri in
let port = Option.map string_of_int (port_of_uri uri) in
let user = uri.Xml.uri_user in
+ let password + match input_password with
+ | None -> Nbdkit.NoPassword
+ | Some ip -> Nbdkit.PasswordFile ip in
- let nbdkit = Nbdkit.create_ssh ?bandwidth ~password:NoPassword ~server
+ let nbdkit = Nbdkit.create_ssh ?bandwidth ~password ~server
?port ?user abs_path in
let qemu_uri = Nbdkit.run nbdkit in
qemu_uri, format
@@ -383,7 +387,7 @@ and find_nics vmx let nics = List.map (fun (_, source)
-> source) nics in
nics
-class input_vmx input_transport arg +class input_vmx input_password
input_transport arg let tmpdir let base_dir = (open_guestfs
())#get_cachedir () in
let t = Mkdtemp.temp_dir ~base_dir "vmx." in
@@ -488,7 +492,7 @@ object
None
| None -> None in
- let disks = find_disks ?bandwidth vmx vmx_source in
+ let disks = find_disks ?bandwidth input_password vmx vmx_source in
let removables = find_removables vmx in
let nics = find_nics vmx in
diff --git a/v2v/input_vmx.mli b/v2v/input_vmx.mli
index 34ec2a5c6..1570a2a93 100644
--- a/v2v/input_vmx.mli
+++ b/v2v/input_vmx.mli
@@ -18,6 +18,6 @@
(** [-i vmx] source. *)
-val input_vmx : [`SSH] option -> string -> Types.input
-(** [input_vmx input_transport arg] sets up an input
+val input_vmx : string option -> [`SSH] option -> string ->
Types.input
+(** [input_vmx input_password input_transport arg] sets up an input
from vmware vmx file. *)
diff --git a/v2v/virt-v2v-input-vmware.pod b/v2v/virt-v2v-input-vmware.pod
index 3acdd773e..11adf1b6c 100644
--- a/v2v/virt-v2v-input-vmware.pod
+++ b/v2v/virt-v2v-input-vmware.pod
@@ -8,6 +8,7 @@ virt-v2v-input-vmware - Using virt-v2v to convert guests from
VMware
virt-v2v -i vmx
-it ssh
+ -ip passwordfile
'ssh://root@esxi.example.com/vmfs/volumes/datastore1/guest/guest.vmx'
[-o* options]
@@ -132,21 +133,21 @@ If the vmx and vmdk files aren't available locally
then you must
I<either> mount the NFS storage on the conversion server I<or>
enable
passwordless SSH on the ESXi hypervisor.
-=head3 VMX: Passwordless SSH using ssh-agent
+=head3 VMX: SSH authentication
-You must also use ssh-agent, and add your ssh public key to
-F</etc/ssh/keys-root/authorized_keys> (on the ESXi hypervisor).
+You can use SSH password authentication, by supplying the name of a
+file containing the password to the I<-ip> option (note this option
+does I<not> take the password directly).
-After doing this, you should check that passwordless access works from
-the virt-v2v server to the ESXi hypervisor. For example:
+If you are not using password authentication, an alternative is to use
+ssh-agent, and add your ssh public key to
+F</etc/ssh/keys-root/authorized_keys> (on the ESXi hypervisor). After
+doing this, you should check that passwordless access works from the
+virt-v2v server to the ESXi hypervisor. For example:
$ ssh root@esxi.example.com
[ logs straight into the shell, no password is requested ]
-Note that password-interactive and Kerberos access are B<not>
-supported. You B<have> to set up ssh access using ssh-agent and
-authorized_keys.
-
=head3 VMX: Construct the SSH URI
When using the SSH input transport you must specify a remote
diff --git a/v2v/virt-v2v-input-xen.pod b/v2v/virt-v2v-input-xen.pod
index 4bb5d2dc2..bafeabf62 100644
--- a/v2v/virt-v2v-input-xen.pod
+++ b/v2v/virt-v2v-input-xen.pod
@@ -5,7 +5,9 @@ virt-v2v-input-xen - Using virt-v2v to convert guests from Xen
=head1 SYNOPSIS
export LIBGUESTFS_BACKEND=direct
- virt-v2v -ic 'xen+ssh://root@xen.example.com' GUEST_NAME [-o* options]
+ virt-v2v -ic 'xen+ssh://root@xen.example.com'
+ -ip passwordfile
+ GUEST_NAME [-o* options]
=head1 DESCRIPTION
@@ -14,24 +16,21 @@ RHEL 5 Xen, or SLES and OpenSUSE Xen hosts.
=head1 INPUT FROM XEN
-=head2 Set up ssh-agent access to Xen host
+=head2 SSH authentication
-Currently you must enable passwordless SSH access to the remote Xen host
-from the virt-v2v conversion server.
+You can use SSH password authentication, by supplying the name of a
+file containing the password to the I<-ip> option (note this option
+does I<not> take the password directly).
-You must also use ssh-agent, and add your ssh public key to
-F</root/.ssh/authorized_keys> (on the Xen host).
-
-After doing this, you should check that passwordless access works
-from the virt-v2v server to the Xen host. For example:
+If you are not using password authentication, an alternative is to use
+ssh-agent, and add your ssh public key to
+F</root/.ssh/authorized_keys> (on the Xen host). After doing this,
+you should check that passwordless access works from the virt-v2v
+server to the Xen host. For example:
$ ssh root@xen.example.com
[ logs straight into the shell, no password is requested ]
-Note that password-interactive and Kerberos access are B<not>
-supported. You B<have> to set up ssh access using ssh-agent and
-authorized_keys.
-
With some modern ssh implementations, legacy crypto policies required
to interoperate with RHEL 5 sshd are disabled. To enable them you may
need to run this command on the conversion server (ie. ssh client),
--
2.23.0
Richard W.M. Jones
2019-Sep-20 09:28 UTC
[Libguestfs] [PATCH v4 12/12] v2v: nbdkit: Add the retry filter unconditionally if it exists.
This experimental filter can be used to work around brief
interruptions in service, such as the network going down, firewalls
timing out connections etc., without requiring virt-v2v to be rerun.
---
v2v/nbdkit.ml | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml
index 776eedce0..4866d3836 100644
--- a/v2v/nbdkit.ml
+++ b/v2v/nbdkit.ml
@@ -168,6 +168,13 @@ let common_create ?bandwidth plugin_name plugin_args
plugin_env )
else [] in
+ (* Retry filter (if it exists) can be used to get around brief
+ * interruptions in service. It must be closest to the plugin.
+ *)
+ if Sys.file_exists (filterdir // "nbdkit-retry-filter.so") then (
+ add_arg "--filter"; add_arg "retry"
+ );
+
let args = get_args () @ [ plugin_name ] @ plugin_args @ rate_args in
{ plugin_name; args; env; dump_config; dump_plugin; filterdir }
--
2.23.0
Martin Kletzander
2019-Sep-20 14:04 UTC
Re: [Libguestfs] [PATCH v4 01/12] v2v: Factor out the nbdkit VDDK code into a new module.
On Fri, Sep 20, 2019 at 10:28:12AM +0100, Richard W.M. Jones wrote:>This refactoring takes the nbdkit-specific code from the ‘-it vddk’ >mode and puts it into a separate module. The idea is to reuse this >module (in future commits) to support ssh, curl and rate limiting. > >This refactoring is not quite neutral because it moves some of the >prechecking into the Nbdkit.create function. This means it will >happen a little bit later in the cycle (in input#source instead of >input#precheck - refer to the diagram in v2v/types.mli). However it's >still almost in the same place and importantly still happens before we >start copying. >--- > v2v/Makefile.am | 2 + > v2v/input_libvirt_vddk.ml | 291 ++++++-------------------------------- > v2v/nbdkit.ml | 280 ++++++++++++++++++++++++++++++++++++ > v2v/nbdkit.mli | 47 ++++++ > 4 files changed, 374 insertions(+), 246 deletions(-) > >diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml >new file mode 100644 >index 000000000..6bf38daa0 >--- /dev/null >+++ b/v2v/nbdkit.ml >@@ -0,0 +1,280 @@ >+(* virt-v2v >+ * Copyright (C) 2009-2019 Red Hat Inc. >+ * >+ * This program is free software; you can redistribute it and/or modify >+ * it under the terms of the GNU General Public License as published by >+ * the Free Software Foundation; either version 2 of the License, or >+ * (at your option) any later version. >+ * >+ * This program is distributed in the hope that it will be useful, >+ * but WITHOUT ANY WARRANTY; without even the implied warranty of >+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+ * GNU General Public License for more details. >+ * >+ * You should have received a copy of the GNU General Public License along >+ * with this program; if not, write to the Free Software Foundation, Inc., >+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ *) >+ >+open Unix >+open Printf >+ >+open Common_gettext.Gettext >+open Std_utils >+open Tools_utils >+open Unix_utils >+ >+open Utils >+ >+type t = { >+ (* The nbdkit plugin name. *) >+ plugin_name : string; >+ >+ (* Parameters (includes the plugin name). *) >+ args : string list; >+ >+ (* Environment variables that may be needed for nbdkit to work. *) >+ env : (string * string) list; >+} >+ >+(* Check that nbdkit is available and new enough. *) >+let error_unless_nbdkit_working () >+ if 0 <> Sys.command "nbdkit --version >/dev/null" then >+ error (f_"nbdkit is not installed or not working"); >+ >+ (* Check it's a new enough version. The latest features we >+ * require are ‘--exit-with-parent’ and ‘--selinux-label’, both >+ * added in 1.1.14. (We use 1.1.16 as the minimum here because >+ * it also adds the selinux=yes|no flag in --dump-config). >+ *) >+ let lines = external_command "nbdkit --help" in >+ let lines = String.concat " " lines in >+ if String.find lines "exit-with-parent" == -1 || >+ String.find lines "selinux-label" == -1 then >+ error (f_"nbdkit is not new enough, you need to upgrade to nbdkit ≥ 1.1.16") >+ >+(* Check that nbdkit was compiled with SELinux support (for the >+ * --selinux-label option). >+ *) >+let error_unless_nbdkit_compiled_with_selinux () >+ if have_selinux then ( >+ let lines = external_command "nbdkit --dump-config" in >+ (* In nbdkit <= 1.1.15 the selinux attribute was not present >+ * at all in --dump-config output so there was no way to tell. >+ * Ignore this case because there will be an error later when >+ * we try to use the --selinux-label parameter. >+ *) >+ if List.mem "selinux=no" (List.map String.trim lines) then >+ error (f_"nbdkit was compiled without SELinux support. You will have to recompile nbdkit with libselinux-devel installed, or else set SELinux to Permissive mode while doing the conversion.") >+ ) >+ >+let common_create plugin_name plugin_args plugin_env >+ error_unless_nbdkit_working (); >+ error_unless_nbdkit_compiled_with_selinux ();Couldn't these (or at least those that do not take any parameters) be checked in a separate ("precheck" again?) function so that the same behaviour is kept (or at least partially)? Even later in the series where you dump the config and use that as an input for some of the checks, it could be done earlier, right? Or is it not the case because it will change based on the environment and that is not ready earlier? It has to be if it was part of precheck before, right? I'm fine with it the way it is, it would just look nicer I think.
Martin Kletzander
2019-Sep-20 14:04 UTC
Re: [Libguestfs] [PATCH v4 07/12] v2v: nbdkit: Add the readahead filter unconditionally if it is available.
On Fri, Sep 20, 2019 at 10:28:18AM +0100, Richard W.M. Jones wrote:>The readahead filter is a self-configuring filter that makes >sequential reads faster when the plugin is slow (and all of the >plugins we use here are always slow). > >I observed the behaviour of the readahead filter with our qcow2 >overlay when converting a guest from a vCenter source. Even when >doing random reads, qemu issues 64K reads which happen to also be the >minimum request size of the readahead filter, so there is no extra >overhead. When doing the sequential copy the readahead filter >performed better than qemu curl’s readahead because it scaled the >prefetched data appropriately on long contiguous stretches and then >shrunk it back to 64K around fragmented parts of the base image. >--- > v2v/nbdkit.ml | 19 ++++++++++++++++++- > 1 file changed, 18 insertions(+), 1 deletion(-) > >diff --git a/v2v/nbdkit.ml b/v2v/nbdkit.ml >index 4bbb8f043..b2d77f963 100644 >--- a/v2v/nbdkit.ml >+++ b/v2v/nbdkit.ml >@@ -49,6 +49,9 @@ type t = { > > (* nbdkit plugin_name --dump-plugin output. *) > dump_plugin : (string * string) list; >+ >+ (* nbdkit directory containing the filters. *) >+ filterdir : string; > } > > (* Check that nbdkit is available and new enough. *) >@@ -105,6 +108,12 @@ let common_create plugin_name plugin_args plugin_env > error_unless_nbdkit_min_version dump_config; > error_unless_nbdkit_compiled_with_selinux dump_config; > >+ (* Get the filterdir. *) >+ let filterdir >+ try List.assoc "filterdir" dump_config >+ with Not_found -> >+ error (f_"nbdkit --dump-config output did not contain filterdir") in >+ > (* Get the nbdkit plugin_name --dump-plugin output, which also > * checks that the plugin is available and loadable. > *) >@@ -133,9 +142,17 @@ let common_create plugin_name plugin_args plugin_env > if have_selinux then ( (* label the socket so qemu can open it *) > add_arg "--selinux-label"; add_arg "system_u:object_r:svirt_socket_t:s0" > ); >+ >+ (* Adding the readahead filter is always a win for our access >+ * patterns. However if it doesn't exist don't worry. >+ *) >+ if Sys.file_exists (filterdir // "nbdkit-readahead-filter.so") then ( >+ add_arg "--filter"; add_arg "readahead"Just out of curiosity, wouldn't it be cleaner, at least in the future, to add an option in nbdkit to report filters it can load? Of course it would require newer nbdkit in the long run.>+ ); >+ > let args = get_args () @ [ plugin_name ] @ plugin_args in > >- { plugin_name; args; env; dump_config; dump_plugin } >+ { plugin_name; args; env; dump_config; dump_plugin; filterdir } > > (* VDDK libraries are located under lib32/ or lib64/ relative to the > * libdir. Note this is unrelated to Linux multilib or multiarch. >-- >2.23.0 > >_______________________________________________ >Libguestfs mailing list >Libguestfs@redhat.com >https://www.redhat.com/mailman/listinfo/libguestfs
Martin Kletzander
2019-Sep-20 14:04 UTC
Re: [Libguestfs] [PATCH v4 00/12] v2v: Change virt-v2v to use nbdkit for input in several modes.
On Fri, Sep 20, 2019 at 10:28:11AM +0100, Richard W.M. Jones wrote:>v3 posted here: >https://www.redhat.com/archives/libguestfs/2019-July/msg00200.html > >v4: > > - The first patch in the v3 series was just a trivial doc whitespace > fix so I pushed it. > > - There's a new patch using the nbdkit-retry-filter. This is not > actually upstream in nbdkit but we know enough about how it will > work. > > - Rebased against master and reran the tests. >That's a lot of code to skim through, but I tried to look at all of it. It makes sense, I could not find any apparent issue in there, but I did not test it. Apart from the comments in particular patches (which are mostly unimportant) I like it.>Rich. > > >_______________________________________________ >Libguestfs mailing list >Libguestfs@redhat.com >https://www.redhat.com/mailman/listinfo/libguestfs
Maybe Matching Threads
- [PATCH v2 00/11] v2v: Change virt-v2v to use nbdkit for input in several modes.
- [PATCH v3 00/12] v2v: Change virt-v2v to use nbdkit for input in several modes.
- [PATCH] v2v: Implement the --bandwidth* options to control network bandwidth.
- [PATCH 00/11] v2v: Change virt-v2v to use nbdkit for input in several modes.
- [PATCH v2v 0/2] v2v: nbdkit: Don't use password=- parameter.