Richard W.M. Jones
2017-Dec-08 16:02 UTC
[Libguestfs] [PATCH v2 0/2] v2v: Add -it vddk and -it ssh flags.
The first patch was previously posted here: https://www.redhat.com/archives/libguestfs/2017-December/msg00018.html That patch hasn't changed except that I made the ‘input_transport’ variable type-safe. The second patch adds a significant new mode for liberating data from VMware: the ability to copy VMs over SSH directly from ESXi hypervisors. Although this requires enabling SSH access (a check-box in the vSphere UI), it is fast, very convenient and doesn't require any proprietary drivers. Rich.
Richard W.M. Jones
2017-Dec-08 16:02 UTC
[Libguestfs] [PATCH v2 1/2] v2v: vddk: Switch to using ‘-it vddk’ to specify VDDK as input transport.
Previously the presence of the ‘--vddk <libdir>’ option magically enabled VDDK mode. However we want to introduce other transports for VMware conversions so this wasn't a very clean choice. With this commit you must use ‘-it vddk’ to specify that you want VDDK as a disk transport. The previous ‘--vddk <libdir>’ option has been renamed to ‘--vddk-libdir <libdir>’ to be consistent with the other passthrough options, and it is no longer required. A new command line looks like: $ export PATH=/path/to/nbdkit:$PATH $ virt-v2v \ -ic 'vpx://root@vcenter.example.com/Datacenter/esxi?no_verify=1' \ | -it vddk \ | --vddk-libdir /path/to/vmware-vix-disklib-distrib \ --vddk-thumbprint xx:xx:xx:... \ "Windows 2003" \ -o local -os /var/tmp where only the two lines marked with ‘|’ have changed. --- v2v/cmdline.ml | 69 +++++++++++++++++++++++++++++------------------ v2v/input_libvirt.ml | 37 ++++++++++++------------- v2v/input_libvirt.mli | 8 +++--- v2v/input_libvirt_vddk.ml | 57 +++++++++++++++++++++++++-------------- v2v/test-v2v-docs.sh | 2 +- v2v/types.ml | 2 +- v2v/types.mli | 2 +- v2v/virt-v2v.pod | 30 +++++++++++++++------ 8 files changed, 126 insertions(+), 81 deletions(-) diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index a452458a1..719e6f057 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -57,15 +57,16 @@ let parse_cmdline () let input_conn = ref None in let input_format = ref None in + let input_transport = ref None in let in_place = ref false in let output_conn = ref None in let output_format = ref None in let output_name = ref None in let output_storage = ref None in let password_file = ref None in - let vddk = ref None in let vddk_config = ref None in let vddk_cookie = ref None in + let vddk_libdir = ref None in let vddk_nfchostport = ref None in let vddk_port = ref None in let vddk_snapshot = ref None in @@ -191,6 +192,8 @@ let parse_cmdline () s_"Libvirt URI"; [ M"if" ], Getopt.String ("format", set_string_option_once "-if" input_format), s_"Input format (for -i disk)"; + [ M"it" ], Getopt.String ("transport", set_string_option_once "-it" input_transport), + s_"Input transport"; [ L"in-place" ], Getopt.Set in_place, s_"Only tune the guest in the input VM"; [ L"machine-readable" ], Getopt.Set machine_readable, @@ -220,12 +223,12 @@ let parse_cmdline () [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o qemu only)"; [ L"root" ], Getopt.String ("ask|... ", set_root_choice), s_"How to choose root filesystem"; - [ L"vddk" ], Getopt.String ("libpath", set_string_option_once "--vddk" vddk), - s_"Use nbdkit VDDK plugin"; [ L"vddk-config" ], Getopt.String ("filename", set_string_option_once "--vddk-config" vddk_config), s_"Set VDDK config file"; [ L"vddk-cookie" ], Getopt.String ("cookie", set_string_option_once "--vddk-cookie" vddk_cookie), s_"Set VDDK cookie"; + [ L"vddk-libdir" ], Getopt.String ("libdir", set_string_option_once "--vddk-libdir" vddk_libdir), + s_"Set VDDK library parent directory"; [ L"vddk-nfchostport" ], Getopt.String ("nfchostport", set_string_option_once "--vddk-nfchostport" vddk_nfchostport), s_"Set VDDK nfchostport"; [ L"vddk-port" ], Getopt.String ("port", set_string_option_once "--vddk-port" vddk_port), @@ -286,6 +289,12 @@ read the man page virt-v2v(1). let input_conn = !input_conn in let input_format = !input_format in let input_mode = !input_mode in + let input_transport + match !input_transport with + | None -> None + | Some "vddk" -> Some `VDDK + | Some transport -> + error (f_"unknown input transport ‘-it %s’") transport in let in_place = !in_place in let machine_readable = !machine_readable in let network_map = !network_map in @@ -303,28 +312,15 @@ read the man page virt-v2v(1). let qemu_boot = !qemu_boot in let root_choice = !root_choice in let vddk_options - match !vddk with - | Some libdir -> - Some { vddk_libdir = libdir; - vddk_config = !vddk_config; - vddk_cookie = !vddk_cookie; - vddk_nfchostport = !vddk_nfchostport; - vddk_port = !vddk_port; - vddk_snapshot = !vddk_snapshot; - vddk_thumbprint = !vddk_thumbprint; - vddk_transports = !vddk_transports; - vddk_vimapiver = !vddk_vimapiver } - | None -> - if !vddk_config <> None || - !vddk_cookie <> None || - !vddk_nfchostport <> None || - !vddk_port <> None || - !vddk_snapshot <> None || - !vddk_thumbprint <> None || - !vddk_transports <> None || - !vddk_vimapiver <> None then - error (f_"‘--vddk-*’ options should only be used when conversion via the nbdkit VDDK plugin has been enabled, ie. using ‘--vddk’."); - None in + { vddk_config = !vddk_config; + vddk_cookie = !vddk_cookie; + vddk_libdir = !vddk_libdir; + vddk_nfchostport = !vddk_nfchostport; + vddk_port = !vddk_port; + vddk_snapshot = !vddk_snapshot; + vddk_thumbprint = !vddk_thumbprint; + vddk_transports = !vddk_transports; + vddk_vimapiver = !vddk_vimapiver } in let vdsm_compat = !vdsm_compat in let vdsm_image_uuids = List.rev !vdsm_image_uuids in let vdsm_vol_uuids = List.rev !vdsm_vol_uuids in @@ -357,6 +353,26 @@ read the man page virt-v2v(1). let password = read_first_line_from_file filename in Some password in + (* Input transport affects whether some parameters should or + * should not be used. + *) + (match input_transport with + | None -> + if !vddk_config <> None || + !vddk_cookie <> None || + !vddk_libdir <> None || + !vddk_nfchostport <> None || + !vddk_port <> None || + !vddk_snapshot <> None || + !vddk_thumbprint <> None || + !vddk_transports <> None || + !vddk_vimapiver <> None then + error (f_"‘--vddk-*’ options should only be used when conversion via the nbdkit VDDK plugin has been enabled, ie. using ‘-it vddk’.") + | Some `VDDK -> + if !vddk_thumbprint = None then + error (f_"‘--vddk-thumbprint’ is required when using ‘-it vddk’.") + ); + (* Parsing of the argument(s) depends on the input mode. *) let input match input_mode with @@ -379,7 +395,8 @@ read the man page virt-v2v(1). | [guest] -> guest | _ -> error (f_"expecting a libvirt guest name on the command line") in - Input_libvirt.input_libvirt vddk_options password input_conn guest + Input_libvirt.input_libvirt vddk_options password + input_conn input_transport guest | `LibvirtXML -> (* -i libvirtxml: Expecting a filename (XML file). *) diff --git a/v2v/input_libvirt.ml b/v2v/input_libvirt.ml index 59845c9b2..66af6371c 100644 --- a/v2v/input_libvirt.ml +++ b/v2v/input_libvirt.ml @@ -27,7 +27,7 @@ open Types open Utils (* Choose the right subclass based on the URI. *) -let input_libvirt vddk_options password libvirt_uri guest +let input_libvirt vddk_options password libvirt_uri input_transport guest match libvirt_uri with | None -> Input_libvirt_other.input_libvirt_other password libvirt_uri guest @@ -39,29 +39,26 @@ let input_libvirt vddk_options password libvirt_uri guest error (f_"could not parse '-ic %s'. Original error message was: %s") orig_uri msg in - match server, scheme with - | None, _ - | Some "", _ (* Not a remote URI. *) + match server, scheme, input_transport with + | None, _, _ + | Some "", _, _ (* Not a remote URI. *) - | Some _, None (* No scheme? *) - | Some _, Some "" -> + | Some _, None, _ (* No scheme? *) + | Some _, Some "", _ -> Input_libvirt_other.input_libvirt_other password libvirt_uri guest - (* vCenter over https, or - * vCenter or ESXi using nbdkit vddk plugin - *) - | Some server, Some ("esx"|"gsx"|"vpx") -> - (match vddk_options with - | None -> - Input_libvirt_vcenter_https.input_libvirt_vcenter_https - password libvirt_uri parsed_uri server guest - | Some vddk_options -> - Input_libvirt_vddk.input_libvirt_vddk vddk_options password - libvirt_uri parsed_uri guest - ) + (* vCenter over https. *) + | Some server, Some ("esx"|"gsx"|"vpx"), None -> + Input_libvirt_vcenter_https.input_libvirt_vcenter_https + password libvirt_uri parsed_uri server guest + + (* vCenter or ESXi using nbdkit vddk plugin *) + | Some server, Some ("esx"|"gsx"|"vpx"), Some `VDDK -> + Input_libvirt_vddk.input_libvirt_vddk vddk_options password + libvirt_uri parsed_uri guest (* Xen over SSH *) - | Some server, Some "xen+ssh" -> + | Some server, Some "xen+ssh", _ -> Input_libvirt_xen_ssh.input_libvirt_xen_ssh password libvirt_uri parsed_uri server guest @@ -71,7 +68,7 @@ let input_libvirt vddk_options password libvirt_uri guest *) (* Unknown remote scheme. *) - | Some _, Some _ -> + | Some _, Some _, _ -> warning (f_"no support for remote libvirt connections to '-ic %s'. The conversion may fail when it tries to read the source disks.") orig_uri; Input_libvirt_other.input_libvirt_other password libvirt_uri guest diff --git a/v2v/input_libvirt.mli b/v2v/input_libvirt.mli index acf2ca417..eb3e48397 100644 --- a/v2v/input_libvirt.mli +++ b/v2v/input_libvirt.mli @@ -18,7 +18,7 @@ (** [-i libvirt] source. *) -val input_libvirt : Types.vddk_options option -> string option -> string option -> string -> Types.input -(** [input_libvirt vddk_options password libvirt_uri guest] creates - and returns a new {!Types.input} object specialized for reading input - from libvirt sources. *) +val input_libvirt : Types.vddk_options -> string option -> string option -> [`VDDK] option -> string -> Types.input +(** [input_libvirt vddk_options password libvirt_uri input_transport guest] + creates and returns a new {!Types.input} object specialized for reading + input from libvirt sources. *) diff --git a/v2v/input_libvirt_vddk.ml b/v2v/input_libvirt_vddk.ml index f3cc6eb9b..1e673f4c7 100644 --- a/v2v/input_libvirt_vddk.ml +++ b/v2v/input_libvirt_vddk.ml @@ -35,11 +35,16 @@ open Printf (* Subclass specialized for handling VMware via nbdkit vddk plugin. *) class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest - (* The VDDK path. *) let libdir = vddk_options.vddk_libdir in - (* Compute the LD_LIBRARY_PATH that we must pass to nbdkit. *) - let library_path = libdir // sprintf "lib%d" Sys.word_size in + + (* VDDK libraries are located under lib32/ or lib64/ relative to the + * libdir. Note this is unrelated to Linux multilib or multiarch. + *) + let libNN = sprintf "lib%d" Sys.word_size in + + (* Compute the LD_LIBRARY_PATH that we may have to pass to nbdkit. *) + let library_path = Option.map (fun libdir -> libdir // libNN) libdir in (* Is SELinux enabled and enforcing on the host? *) let have_selinux @@ -47,18 +52,25 @@ class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest (* Check that the VDDK path looks reasonable. *) let error_unless_vddk_libdir () - if not (is_directory libdir) then - error (f_"‘--vddk %s’ does not point to a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libdir; + (match libdir with + | None -> () + | Some libdir -> + if not (is_directory libdir) then + error (f_"‘--vddk-libdir %s’ does not point to a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libdir + ); - if not (is_directory library_path) then - error (f_"VDDK library path %s not found or not a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") - library_path + (match library_path with + | None -> () + | Some library_path -> + if not (is_directory library_path) then + error (f_"VDDK library path %s not found or not a directory. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") library_path + ) in (* Check that nbdkit is available and new enough. *) let error_unless_nbdkit_working () if 0 <> Sys.command "nbdkit --version >/dev/null" then - error (f_"nbdkit is not installed or not working. It is required to use ‘--vddk’. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual."); + error (f_"nbdkit is not installed or not working. It is required to use ‘-it vddk’. See \"INPUT FROM VDDK\" in the virt-v2v(1) manual."); (* Check it's a new enough version. The latest features we * require are ‘--exit-with-parent’ and ‘--selinux-label’, both @@ -74,14 +86,20 @@ class input_libvirt_vddk vddk_options password libvirt_uri parsed_uri guest (* Check that the VDDK plugin is installed and working *) let error_unless_nbdkit_vddk_working () + let set_ld_library_path + match library_path with + | None -> "" + | Some library_path -> + sprintf "LD_LIBRARY_PATH=%s " (quote library_path) in + let cmd - sprintf "LD_LIBRARY_PATH=%s nbdkit vddk --dump-plugin >/dev/null" - (quote library_path) in + sprintf "%snbdkit vddk --dump-plugin >/dev/null" + set_ld_library_path in if Sys.command cmd <> 0 then ( (* See if we can diagnose why ... *) let cmd - sprintf "LD_LIBRARY_PATH=%s LANG=C nbdkit vddk --dump-plugin 2>&1 | grep -sq libvixDiskLib.so" - (quote library_path) in + sprintf "LANG=C %snbdkit vddk --dump-plugin 2>&1 | grep -sq libvixDiskLib.so" + set_ld_library_path in let needs_library = Sys.command cmd = 0 in if not needs_library then error (f_"nbdkit VDDK plugin is not installed or not working. It is required if you want to use VDDK. @@ -92,9 +110,9 @@ See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") else error (f_"nbdkit VDDK plugin is not installed or not working. It is required if you want to use VDDK. -It looks like you did not set the right path in the ‘--vddk’ option, or your copy of the VDDK directory is incomplete. There should be a library called ’%s/libvixDiskLib.so.?’. +It looks like you did not set the right path in the ‘--vddk-libdir’ option, or your copy of the VDDK directory is incomplete. There should be a library called ’<libdir>/%s/libvixDiskLib.so.?’. -See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") library_path +See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") libNN ) in @@ -121,6 +139,7 @@ See also \"INPUT FROM VDDK\" in the virt-v2v(1) manual.") library_path let vddk_passthrus [ "config", (fun { vddk_config } -> vddk_config); "cookie", (fun { vddk_cookie } -> vddk_cookie); + "libdir", (fun { vddk_libdir } -> vddk_libdir); "nfchostport", (fun { vddk_nfchostport } -> vddk_nfchostport); "port", (fun { vddk_port } -> vddk_port); "snapshot", (fun { vddk_snapshot } -> vddk_snapshot); @@ -149,9 +168,8 @@ object | Some field -> sprintf " --vddk-%s %s" name field ) vddk_passthrus ) in - sprintf "%s --vddk %s%s" + sprintf "%s -it vddk %s" super#as_options (* superclass prints "-i libvirt etc" *) - vddk_options.vddk_libdir pt_options method source () @@ -252,7 +270,6 @@ object add_arg (sprintf "user=%s" user); add_arg password_param; add_arg (sprintf "vm=moref=%s" moref); - add_arg (sprintf "libdir=%s" libdir); (* The passthrough parameters. *) List.iter ( @@ -297,7 +314,7 @@ object (* Print the full command we are about to run when debugging. *) if verbose () then ( eprintf "running nbdkit:\n"; - eprintf "LD_LIBRARY_PATH=%s" library_path; + Option.may (eprintf "LD_LIBRARY_PATH=%s") library_path; List.iter (fun arg -> eprintf " %s" (quote arg)) args; prerr_newline () ); @@ -310,7 +327,7 @@ object let pid = fork () in if pid = 0 then ( (* Child process (nbdkit). *) - putenv "LD_LIBRARY_PATH" library_path; + Option.may (putenv "LD_LIBRARY_PATH") library_path; execvp "nbdkit" args ); diff --git a/v2v/test-v2v-docs.sh b/v2v/test-v2v-docs.sh index 8bd03f68e..5d034c465 100755 --- a/v2v/test-v2v-docs.sh +++ b/v2v/test-v2v-docs.sh @@ -22,4 +22,4 @@ $TEST_FUNCTIONS skip_if_skipped $top_srcdir/podcheck.pl virt-v2v.pod virt-v2v \ - --ignore=--debug-overlay,--ic,--if,--no-trim,--oa,--oc,--of,--on,--os,--vmtype + --ignore=--debug-overlay,--ic,--if,--it,--no-trim,--oa,--oc,--of,--on,--os,--vmtype diff --git a/v2v/types.ml b/v2v/types.ml index ee6b642bf..be86f1b3b 100644 --- a/v2v/types.ml +++ b/v2v/types.ml @@ -480,9 +480,9 @@ type root_choice = AskRoot | SingleRoot | FirstRoot | RootDev of string type output_allocation = Sparse | Preallocated type vddk_options = { - vddk_libdir : string; vddk_config : string option; vddk_cookie : string option; + vddk_libdir : string option; vddk_nfchostport : string option; vddk_port : string option; vddk_snapshot : string option; diff --git a/v2v/types.mli b/v2v/types.mli index a426bd696..95d890638 100644 --- a/v2v/types.mli +++ b/v2v/types.mli @@ -340,9 +340,9 @@ type output_allocation = Sparse | Preallocated (** Type of [-oa] (output allocation) option. *) type vddk_options = { - vddk_libdir : string; vddk_config : string option; vddk_cookie : string option; + vddk_libdir : string option; vddk_nfchostport : string option; vddk_port : string option; vddk_snapshot : string option; diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index 4ac44988e..503830c9d 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -379,6 +379,13 @@ See L</IN PLACE CONVERSION> below. Conflicts with all I<-o *> options. +=item B<-it> B<vddk> + +Use VMware VDDK as a transport to copy the input disks. See +L</INPUT FROM VDDK> below. If you use this parameter then you may +need to use other I<--vddk*> options to specify how to connect through +VDDK. + =item B<--keys-from-stdin> Read key or passphrase parameters from stdin. The default is @@ -633,13 +640,20 @@ boot an operating system from the first virtio disk. Specifically, F</boot> must be on the first virtio disk, and it cannot chainload an OS which is not in the first virtio disk. -=item B<--vddk> LIBDIR +=item B<--vddk-libdir> LIBDIR -Enable VDDK input from VMware vCenter or ESXi. C<LIBDIR> is the top -directory of the VDDK library. This directory should I<contain> +Set the VDDK library directory. This directory should I<contain> subdirectories called F<include>, F<lib64> etc., but do not include F<lib64> actually in the parameter. +In most cases this parameter is required when using the I<-it vddk> +(VDDK) transport. See L</INPUT FROM VDDK> below for details. + +=item B<--vddk-thumbprint> xx:xx:xx:... + +Set the thumbprint of the remote VMware server. + +This parameter is required when using the I<-it vddk> (VDDK) transport. See L</INPUT FROM VDDK> below for details. =item B<--vddk-config> FILENAME @@ -652,16 +666,13 @@ See L</INPUT FROM VDDK> below for details. =item B<--vddk-snapshot> SNAPSHOT-MOREF -=item B<--vddk-thumbprint> xx:xx:xx:... - =item B<--vddk-transports> MODE:MODE:... =item B<--vddk-vimapiver> APIVER When using VDDK mode, these options are passed unmodified to the L<nbdkit(1)> VDDK plugin. Please refer to L<nbdkit-vddk-plugin(1)>. -If I<--vddk> is present, I<--vddk-thumbprint> is also required, -the rest are optional. +These are all optional. =item B<--vdsm-compat=0.10> @@ -1543,6 +1554,8 @@ continuing. =head2 VDDK: IMPORTING A GUEST +The I<-it vddk> parameter selects VDDK as the input transport for disks. + To import a particular guest from vCenter server or ESXi hypervisor, use a command like the following, substituting the URI, guest name and SSL thumbprint: @@ -1550,7 +1563,8 @@ SSL thumbprint: $ export PATH=/path/to/nbdkit:$PATH $ virt-v2v \ -ic 'vpx://root@vcenter.example.com/Datacenter/esxi?no_verify=1' \ - --vddk /path/to/vmware-vix-disklib-distrib \ + -it vddk \ + --vddk-libdir /path/to/vmware-vix-disklib-distrib \ --vddk-thumbprint xx:xx:xx:... \ "Windows 2003" \ -o local -os /var/tmp -- 2.13.2
Richard W.M. Jones
2017-Dec-08 16:02 UTC
[Libguestfs] [PATCH v2 2/2] v2v: -i vmx: Enhance VMX support with ability to use ‘-it ssh’ transport.
This enhances the existing VMX input support allowing it to be used over SSH to the ESXi server. The original command (for local .vmx files) was: $ virt-v2v -i vmx guest.vmx -o local -os /var/tmp Adding ‘-it ssh’ and using an SSH remote path gives the new syntax: $ virt-v2v \ -i vmx -it ssh \ "root@esxi.example.com:/vmfs/volumes/datastore1/guest/guest.vmx" \ -o local -os /var/tmp I anticipate that this input method will be widely used enough that it deserves its own example at the top of the man page. --- v2v/cmdline.ml | 26 +++++-- v2v/input_libvirt_other.ml | 9 --- v2v/input_libvirt_other.mli | 1 - v2v/input_vmx.ml | 171 +++++++++++++++++++++++++++++++++++++------- v2v/input_vmx.mli | 5 +- v2v/utils.ml | 9 +++ v2v/utils.mli | 2 + v2v/virt-v2v.pod | 91 ++++++++++++++++++----- 8 files changed, 254 insertions(+), 60 deletions(-) diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml index 719e6f057..0c6af3ed8 100644 --- a/v2v/cmdline.ml +++ b/v2v/cmdline.ml @@ -292,6 +292,7 @@ read the man page virt-v2v(1). let input_transport match !input_transport with | None -> None + | Some "ssh" -> Some `SSH | Some "vddk" -> Some `VDDK | Some transport -> error (f_"unknown input transport ‘-it %s’") transport in @@ -357,7 +358,8 @@ read the man page virt-v2v(1). * should not be used. *) (match input_transport with - | None -> + | None + | Some `SSH -> if !vddk_config <> None || !vddk_cookie <> None || !vddk_libdir <> None || @@ -395,6 +397,12 @@ read the man page virt-v2v(1). | [guest] -> guest | _ -> error (f_"expecting a libvirt guest name on the command line") in + let input_transport + match input_transport with + | None -> None + | Some `VDDK -> Some `VDDK + | Some `SSH -> + error (f_"only ‘-it vddk’ can be used here") in Input_libvirt.input_libvirt vddk_options password input_conn input_transport guest @@ -417,13 +425,19 @@ read the man page virt-v2v(1). Input_ova.input_ova filename | `VMX -> - (* -i vmx: Expecting an vmx filename. *) - let filename + (* -i vmx: Expecting a vmx filename or SSH remote path. *) + let arg match args with - | [filename] -> filename + | [arg] -> arg | _ -> - error (f_"expecting a VMX file name on the command line") in - Input_vmx.input_vmx filename in + error (f_"expecting a single VMX file name or SSH remote path on the command line") in + let input_transport + match input_transport with + | None -> None + | Some `SSH -> Some `SSH + | Some `VDDK -> + error (f_"only ‘-it ssh’ can be used here") in + Input_vmx.input_vmx input_transport arg in (* Common error message. *) let error_option_cannot_be_used_in_output_mode mode opt diff --git a/v2v/input_libvirt_other.ml b/v2v/input_libvirt_other.ml index e08d79cc9..42d8c7df6 100644 --- a/v2v/input_libvirt_other.ml +++ b/v2v/input_libvirt_other.ml @@ -39,15 +39,6 @@ let error_if_libvirt_does_not_support_json_backingfile () Libvirt_utils.libvirt_get_version () < (2, 1, 0) then error (f_"because of libvirt bug https://bugzilla.redhat.com/1134878 you must EITHER upgrade to libvirt >= 2.1.0 OR set this environment variable:\n\nexport LIBGUESTFS_BACKEND=direct\n\nand then rerun the virt-v2v command.") -(* xen+ssh URLs use the SSH driver in CURL. Currently this requires - * ssh-agent authentication. Give a clear error if this hasn't been - * set up (RHBZ#1139973). - *) -let error_if_no_ssh_agent () - try ignore (Sys.getenv "SSH_AUTH_SOCK") - with Not_found -> - error (f_"ssh-agent authentication has not been set up ($SSH_AUTH_SOCK is not set). Please read \"INPUT FROM RHEL 5 XEN\" in the virt-v2v(1) man page.") - (* Superclass. *) class virtual input_libvirt (password : string option) libvirt_uri guest object diff --git a/v2v/input_libvirt_other.mli b/v2v/input_libvirt_other.mli index 494ca908a..8b1e8aa1d 100644 --- a/v2v/input_libvirt_other.mli +++ b/v2v/input_libvirt_other.mli @@ -19,7 +19,6 @@ (** [-i libvirt] source. *) val error_if_libvirt_does_not_support_json_backingfile : unit -> unit -val error_if_no_ssh_agent : unit -> unit class virtual input_libvirt : string option -> string option -> string -> object method precheck : unit -> unit diff --git a/v2v/input_vmx.ml b/v2v/input_vmx.ml index c50217b9e..3032eba96 100644 --- a/v2v/input_vmx.ml +++ b/v2v/input_vmx.ml @@ -21,14 +21,82 @@ open Scanf open Std_utils open Tools_utils +open Unix_utils open Common_gettext.Gettext open Types open Utils open Name_from_disk -let rec find_disks vmx vmx_filename - find_scsi_disks vmx vmx_filename @ find_ide_disks vmx vmx_filename +type vmx_source + | File of string (* local file or NFS *) + | SSH of string option * string * string (* SSH username, server, path *) + +(* The single filename on the command line is intepreted either as + * a local file or a remote SSH path (only if ‘-it ssh’). + *) +let vmx_source_of_arg input_transport arg + match input_transport, arg with + | None, arg -> File arg + | Some `SSH, arg -> + let arg1, path = String.split ":" arg in + if path = "" then + error (f_"expecting [user@]server:path with ‘-it ssh’"); + let user, server = match String.split "@" arg1 with + | server, "" -> None, server + | user, server -> Some user, server in + SSH (user, server, path) + +(* 'scp' a remote file into a temporary local file, returning the path + * of the temporary local file. + *) +let memo_tmpdir = ref None +let scp_from_remote_to_temporary user server path filename + let tmpdir + match !memo_tmpdir with + | None -> + let base_dir = (open_guestfs ())#get_cachedir () in + let t = Mkdtemp.temp_dir ~base_dir "vmx." in + rmdir_on_exit t; + memo_tmpdir := Some t; + t + | Some tmpdir -> tmpdir in + + let localfile = tmpdir // filename in + + (* XXX Assumes default port number. *) + let cmd + sprintf "scp%s %s%s:%s %s" + (if verbose () then "" else " -q") + (match user with None -> "" | Some user -> quote user ^ "@") + (quote server) + (* The double quoting of the path is counter-intuitive + * but correct, see: + * https://stackoverflow.com/questions/19858176/how-to-escape-spaces-in-path-during-scp-copy-in-linux + *) + (quote (quote path)) + (quote localfile) in + if verbose () then + eprintf "%s\n%!" cmd; + if Sys.command cmd <> 0 then + error (f_"could not copy the VMX file from the remote server, see earlier error messages"); + localfile + +(* Test if [path] exists on the remote server. *) +let remote_file_exists user server path + (* XXX Assumes default port number. *) + let cmd + sprintf "ssh %s%s test -f %s" + (match user with None -> "" | Some user -> quote user ^ "@") + (quote server) + (* Double quoting is necessary here, see above. *) + (quote (quote path)) in + if verbose () then + eprintf "%s\n%!" cmd; + Sys.command cmd = 0 + +let rec find_disks vmx vmx_source + find_scsi_disks vmx vmx_source @ find_ide_disks vmx vmx_source (* Find all SCSI hard disks. * @@ -38,7 +106,7 @@ let rec find_disks vmx vmx_filename * | omitted * scsi0:0.fileName = "guest.vmdk" *) -and find_scsi_disks vmx vmx_filename +and find_scsi_disks vmx vmx_source let get_scsi_controller_target ns sscanf ns "scsi%d:%d" (fun c t -> c, t) in @@ -50,7 +118,7 @@ and find_scsi_disks vmx vmx_filename Some "scsi-harddisk"; None ] in let scsi_controller = Source_SCSI in - find_hdds vmx vmx_filename + find_hdds vmx vmx_source get_scsi_controller_target is_scsi_controller_target scsi_device_types scsi_controller @@ -60,7 +128,7 @@ and find_scsi_disks vmx vmx_filename * ide0:0.deviceType = "ata-hardDisk" * ide0:0.fileName = "guest.vmdk" *) -and find_ide_disks vmx vmx_filename +and find_ide_disks vmx vmx_source let get_ide_controller_target ns sscanf ns "ide%d:%d" (fun c t -> c, t) in @@ -71,11 +139,11 @@ and find_ide_disks vmx vmx_filename let ide_device_types = [ Some "ata-harddisk" ] in let ide_controller = Source_IDE in - find_hdds vmx vmx_filename + find_hdds vmx vmx_source get_ide_controller_target is_ide_controller_target ide_device_types ide_controller -and find_hdds vmx vmx_filename +and find_hdds vmx vmx_source get_controller_target is_controller_target device_types controller (* Find namespaces matching '(ide|scsi)X:Y' with suitable deviceType. *) @@ -101,9 +169,9 @@ and find_hdds vmx vmx_filename match path, v with | [ns; "filename"], Some filename -> let c, t = get_controller_target ns in + let uri, format = qemu_uri_of_filename vmx_source filename in let s = { s_disk_id = (-1); - s_qemu_uri = qemu_uri_of_filename vmx_filename filename; - s_format = Some "vmdk"; + s_qemu_uri = uri; s_format = Some format; s_controller = Some controller } in Some (c, t, s) | _ -> None @@ -125,17 +193,48 @@ and find_hdds vmx vmx_filename (* The filename can be an absolute path, but is more often a * path relative to the location of the vmx file. * - * Note that we always end up with an absolute path, which is - * also useful because it means we won't have any paths that - * could be misinterpreted by qemu. + * This constructs a QEMU URI of the filename relative to the + * vmx file (which might be remote over SSH). *) -and qemu_uri_of_filename vmx_filename filename - if not (Filename.is_relative filename) then - filename - else ( - let dir = Filename.dirname (absolute_path vmx_filename) in - dir // filename - ) +and qemu_uri_of_filename vmx_source filename + match vmx_source with + | File vmx_filename -> + (* Always ensure this returns an absolute path to avoid + * any confusion with filenames containing colons. + *) + absolute_path_from_other_file vmx_filename filename, "vmdk" + + | SSH (user, server, vmx_path) -> + let abs_path = absolute_path_from_other_file vmx_path filename in + let format = "vmdk" in + + (* XXX This is a hack to work around qemu / VMDK limitation + * "Cannot use relative extent paths with VMDK descriptor file" + * We can remove this if the above is fixed. + *) + let abs_path, format + let flat_vmdk + PCRE.replace (PCRE.compile "\\.vmdk$") "-flat.vmdk" abs_path in + if remote_file_exists user server flat_vmdk then (flat_vmdk, "raw") + else (abs_path, format) in + + let json_params = [ + "file.driver", JSON.String "ssh"; + "file.path", JSON.String abs_path; + "file.host", JSON.String server; + "file.host_key_check", JSON.String "no"; + ] in + let json_params + match user with + | None -> json_params + | Some user -> + ("file.user", JSON.String user) :: json_params in + + "json:" ^ JSON.string_of_doc json_params, format + +and absolute_path_from_other_file other_filename filename + if not (Filename.is_relative filename) then filename + else (Filename.dirname (absolute_path other_filename)) // filename (* Find all removable disks. * @@ -268,21 +367,41 @@ and find_nics vmx let nics = List.map (fun (_, source) -> source) nics in nics -class input_vmx vmx_filename = object +class input_vmx input_transport arg = object inherit input - method as_options = "-i vmx " ^ vmx_filename + method as_options = "-i vmx " ^ arg + + method precheck () + match input_transport with + | None -> () + | Some `SSH -> + if backend_is_libvirt () then + error (f_"because libvirtd doesn't pass the SSH_AUTH_SOCK environment variable to qemu you must set this environment variable:\n\nexport LIBGUESTFS_BACKEND=direct\n\nand then rerun the virt-v2v command."); + error_if_no_ssh_agent () method source () - (* Parse the VMX file. *) - let vmx = Parse_vmx.parse_file vmx_filename in + let vmx_source = vmx_source_of_arg input_transport arg in + + (* If the transport is SSH, fetch the file from remote, else + * parse it from local. + *) + let vmx + match vmx_source with + | File filename -> Parse_vmx.parse_file filename + | SSH (user, server, path) -> + let filename + scp_from_remote_to_temporary user server path "source.vmx" in + Parse_vmx.parse_file filename in let name match Parse_vmx.get_string vmx ["displayName"] with + | Some s -> s | None -> warning (f_"no displayName key found in VMX file"); - name_from_disk vmx_filename - | Some s -> s in + match vmx_source with + | File filename -> name_from_disk filename + | SSH (_, _, path) -> name_from_disk path in let memory_mb match Parse_vmx.get_int64 vmx ["memSize"] with @@ -333,7 +452,7 @@ class input_vmx vmx_filename = object None | None -> None in - let disks = find_disks vmx vmx_filename in + let disks = find_disks vmx vmx_source in let removables = find_removables vmx in let nics = find_nics vmx in diff --git a/v2v/input_vmx.mli b/v2v/input_vmx.mli index f236f8716..34ec2a5c6 100644 --- a/v2v/input_vmx.mli +++ b/v2v/input_vmx.mli @@ -18,5 +18,6 @@ (** [-i vmx] source. *) -val input_vmx : string -> Types.input -(** [input_vmx filename] sets up an input from vmware vmx file. *) +val input_vmx : [`SSH] option -> string -> Types.input +(** [input_vmx input_transport arg] sets up an input + from vmware vmx file. *) diff --git a/v2v/utils.ml b/v2v/utils.ml index 91c0ed1c8..d5d177e42 100644 --- a/v2v/utils.ml +++ b/v2v/utils.ml @@ -138,6 +138,15 @@ let backend_is_libvirt () let backend = fst (String.split ":" backend) in backend = "libvirt" +(* When using the SSH driver in qemu (currently) this requires + * ssh-agent authentication. Give a clear error if this hasn't been + * set up (RHBZ#1139973). This might improve if we switch to libssh1. + *) +let error_if_no_ssh_agent () + try ignore (Sys.getenv "SSH_AUTH_SOCK") + with Not_found -> + error (f_"ssh-agent authentication has not been set up ($SSH_AUTH_SOCK is not set). This is required by qemu to do passwordless ssh access. See the virt-v2v(1) man page for more information.") + let ws = PCRE.compile "\\s+" let find_file_in_tar tar filename diff --git a/v2v/utils.mli b/v2v/utils.mli index 8d902a53a..422fde298 100644 --- a/v2v/utils.mli +++ b/v2v/utils.mli @@ -53,6 +53,8 @@ val qemu_img_supports_offset_and_size : unit -> bool val backend_is_libvirt : unit -> bool (** Return true iff the current backend is libvirt. *) +val error_if_no_ssh_agent : unit -> unit + val find_file_in_tar : string -> string -> int64 * int64 (** [find_file_in_tar tar filename] looks up file in [tar] archive and returns a tuple containing at which byte it starts and how long the file is. diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod index 503830c9d..79269a9da 100644 --- a/v2v/virt-v2v.pod +++ b/v2v/virt-v2v.pod @@ -118,6 +118,23 @@ Note that after conversion, the guest will appear in the RHV-M Export Storage Domain, from where you will need to import it using the RHV-M user interface. (See L</OUTPUT TO RHV>). +=head2 Convert from ESXi hypervisor over SSH to local libvirt + +You have an ESXi hypervisor called C<esxi.example.com> with SSH access +enabled. You want to convert from VMFS storage on that server to +a local file. + + virt-v2v \ + -i vmx -it ssh \ + "root@esxi.example.com:/vmfs/volumes/datastore1/guest/guest.vmx" \ + -o local -os /var/tmp + +The guest must not be running. Virt-v2v would I<not> need to be run +as root in this case. + +For more information about converting from VMX files see +L</INPUT FROM VMWARE VMX> below. + =head2 Convert disk image to OpenStack glance Given a disk image from another hypervisor that you want to convert to @@ -343,9 +360,10 @@ L</INPUT FROM VMWARE OVA> below Set the input method to I<vmx>. -In this mode you can read a VMware vmx file directly. This is useful -when VMware VMs are stored on an NFS server which you can mount -directly. See L</INPUT FROM VMWARE VMX> below +In this mode you can read a VMware vmx file directly or over SSH. +This is useful when VMware VMs are stored on an NFS server which you +can mount directly, or where you have access by SSH to an ESXi +hypervisor. See L</INPUT FROM VMWARE VMX> below =item B<-ic> libvirtURI @@ -379,6 +397,11 @@ See L</IN PLACE CONVERSION> below. Conflicts with all I<-o *> options. +=item B<-it> B<ssh> + +When using I<-i vmx>, this enables the ssh transport. +See L</INPUT FROM VMWARE VMX> below. + =item B<-it> B<vddk> Use VMware VDDK as a transport to copy the input disks. See @@ -1347,9 +1370,23 @@ directory containing the files: =head1 INPUT FROM VMWARE VMX -Virt-v2v is able to import guests from VMware’s vmx files. This is -useful where VMware virtual machines are stored on a separate NFS -server and you are able to mount the NFS storage directly. +Virt-v2v is able to import guests from VMware’s vmx files. + +This is useful in two cases: + +=over 4 + +=item 1. + +VMware virtual machines are stored on a separate NFS server and you +are able to mount the NFS storage directly. + +=item 2. + +You have enabled SSH access to the VMware ESXi hypervisor and there is +a C</vmfs/volumes> folder containing the virtual machines. + +=back If you find a folder of files called F<I<guest>.vmx>, F<I<guest>.vmxf>, F<I<guest>.nvram> and one or more F<.vmdk> disk @@ -1375,28 +1412,50 @@ With other methods, virt-v2v tries to prevent concurrent access, but because the I<-i vmx> method works directly against the storage, checking for concurrent access is not possible. -=head2 VMX: MOUNT THE NFS STORAGE ON THE CONVERSION SERVER +=head2 VMX: ACCESS TO THE STORAGE CONTAINING THE VMX AND VMDK FILES -Virt-v2v must be able to access the F<.vmx> file and any local -F<.vmdk> disks. Normally this means you must mount the NFS storage -containing these files. +If the vmx and vmdk files aren't available locally then you must +I<either> mount the NFS storage on the conversion server I<or> enable +passwordless SSH on the ESXi hypervisor. + +=head3 VMX: Passwordless SSH using ssh-agent + +You must also use ssh-agent, and add your ssh public key to +F</etc/ssh/keys-root/authorized_keys> (on the ESXi hypervisor). + +After doing this, you should check that passwordless access works from +the virt-v2v server to the ESXi hypervisor. For example: + + $ ssh root@esxi.example.com + [ logs straight into the shell, no password is requested ] + +Note that password-interactive and Kerberos access are B<not> +supported. You B<have> to set up ssh access using ssh-agent and +authorized_keys. =head2 VMX: IMPORTING A GUEST -To import a vmx file, do: +To import a vmx file from a local file or NFS, do: $ virt-v2v -i vmx guest.vmx -o local -os /var/tmp +To import a vmx file over SSH, add I<-it ssh> to select the SSH +transport and supply a remote C<server:/path> with optional username: + + $ virt-v2v \ + -i vmx -it ssh \ + "root@esxi.example.com:/vmfs/volumes/datastore1/guest/guest.vmx" \ + -o local -os /var/tmp + Virt-v2v processes the vmx file and uses it to find the location of any vmdk disks. =head1 INPUT FROM VMWARE ESXi HYPERVISOR -Virt-v2v cannot access an ESXi hypervisor directly. You should use -the OVA or VMX methods above (see L</INPUT FROM VMWARE OVA> and/or -L</INPUT FROM VMWARE VMX>) if possible, as it is much faster and -requires much less disk space than the method described in this -section. +You should use the OVA or VMX methods above (see L</INPUT FROM VMWARE +OVA> and/or L</INPUT FROM VMWARE VMX>) if possible, as it is much +faster and requires much less disk space than the method described in +this section. You can use the L<virt-v2v-copy-to-local(1)> tool to copy the guest off the hypervisor into a local file, and then convert it. -- 2.13.2
Pino Toscano
2017-Dec-08 16:55 UTC
Re: [Libguestfs] [PATCH v2 1/2] v2v: vddk: Switch to using ‘-it vddk’ to specify VDDK as input transport.
On Friday, 8 December 2017 17:02:29 CET Richard W.M. Jones wrote:> Previously the presence of the ‘--vddk <libdir>’ option magically > enabled VDDK mode. However we want to introduce other transports for > VMware conversions so this wasn't a very clean choice. > > With this commit you must use ‘-it vddk’ to specify that you want VDDK > as a disk transport. The previous ‘--vddk <libdir>’ option has been > renamed to ‘--vddk-libdir <libdir>’ to be consistent with the other > passthrough options, and it is no longer required. > > A new command line looks like: > > $ export PATH=/path/to/nbdkit:$PATH > $ virt-v2v \ > -ic 'vpx://root@vcenter.example.com/Datacenter/esxi?no_verify=1' \ > | -it vddk \ > | --vddk-libdir /path/to/vmware-vix-disklib-distrib \ > --vddk-thumbprint xx:xx:xx:... \ > "Windows 2003" \ > -o local -os /var/tmp > > where only the two lines marked with ‘|’ have changed. > --- > [...] > @@ -286,6 +289,12 @@ read the man page virt-v2v(1). > let input_conn = !input_conn in > let input_format = !input_format in > let input_mode = !input_mode in > + let input_transport > + match !input_transport with > + | None -> None > + | Some "vddk" -> Some `VDDK > + | Some transport -> > + error (f_"unknown input transport ‘-it %s’") transport inAnother option could be to switch from a polymorphic variant to a simple type, e.g.: type input_transport | None | VDDK of vddk_options | SSH So it will remove the extra vddk_options parameters in most functions where input_transport is passed, and easily support more options for future transports. The rest of the changes seem fine. -- Pino Toscano
Pino Toscano
2017-Dec-08 17:04 UTC
Re: [Libguestfs] [PATCH v2 2/2] v2v: -i vmx: Enhance VMX support with ability to use ‘-it ssh’ transport.
On Friday, 8 December 2017 17:02:30 CET Richard W.M. Jones wrote:> This enhances the existing VMX input support allowing it to be > used over SSH to the ESXi server. > > The original command (for local .vmx files) was: > > $ virt-v2v -i vmx guest.vmx -o local -os /var/tmp > > Adding ‘-it ssh’ and using an SSH remote path gives the new syntax: > > $ virt-v2v \ > -i vmx -it ssh \ > "root@esxi.example.com:/vmfs/volumes/datastore1/guest/guest.vmx" \ > -o local -os /var/tmp > > I anticipate that this input method will be widely used enough that it > deserves its own example at the top of the man page. > --- > [...] > +(* The single filename on the command line is intepreted either as > + * a local file or a remote SSH path (only if ‘-it ssh’). > + *) > +let vmx_source_of_arg input_transport arg > + match input_transport, arg with > + | None, arg -> File arg > + | Some `SSH, arg -> > + let arg1, path = String.split ":" arg in > + if path = "" then > + error (f_"expecting [user@]server:path with ‘-it ssh’"); > + let user, server = match String.split "@" arg1 with > + | server, "" -> None, server > + | user, server -> Some user, server in > + SSH (user, server, path)IMHO this new transport could use the standard URI syntax, so root@esxi.example.com/vmfs/volumes/datastore1/guest/guest.vmx instead of root@esxi.example.com:/vmfs/volumes/datastore1/guest/guest.vmx This way, Xml.parse_uri can be used to parse it, giving a better code for extracting all the parts (including the port, for example). The rest seems fine, at a quick glance. -- Pino Toscano
Apparently Analagous Threads
- [PATCH] mllib: Bind %identity C primitive to Common_utils.identity.
- [PATCH] v2v: Implement SSH password authentication for Xen and VMX over SSH.
- [PATCH v2 0/2] v2v: Add -it vddk and -it ssh flags.
- [PATCH v2 00/11] v2v: Change virt-v2v to use nbdkit for input in several modes.
- [PATCH] v2v: Implement the --bandwidth* options to control network bandwidth.