Richard W.M. Jones
2018-Aug-14 17:27 UTC
[Libguestfs] [PATCH] v2v: Add --print-estimate option to print source size estimate.
This option prints the estimated size of the data that will be copied
from the source disk.
For interest, the test prints:
3747840 ../test-data/phony-guests/windows.img
Estimate: 3710976
---
v2v/Makefile.am | 2 ++
v2v/cmdline.ml | 17 ++++++++++--
v2v/cmdline.mli | 2 ++
v2v/test-v2v-print-estimate.sh | 47 +++++++++++++++++++++++++++++++++
v2v/v2v.ml | 48 ++++++++++++++++++++++++++++++++++
v2v/virt-v2v.pod | 17 ++++++++++++
6 files changed, 131 insertions(+), 2 deletions(-)
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 5461055d1..2ea0dd1d5 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -379,6 +379,7 @@ TESTS += \
test-v2v-oa-option.sh \
test-v2v-of-option.sh \
test-v2v-on-option.sh \
+ test-v2v-print-estimate.sh \
test-v2v-print-source.sh \
test-v2v-sound.sh \
$(SLOW_TESTS) \
@@ -534,6 +535,7 @@ EXTRA_DIST += \
test-v2v-oa-option.sh \
test-v2v-of-option.sh \
test-v2v-on-option.sh \
+ test-v2v-print-estimate.sh \
test-v2v-print-source.expected \
test-v2v-print-source.sh \
test-v2v-print-source.xml \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 5b2df3555..74cc27714 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -33,10 +33,12 @@ type cmdline = {
debug_overlays : bool;
do_copy : bool;
in_place : bool;
+ machine_readable : bool;
network_map : Networks.t;
output_alloc : output_allocation;
output_format : string option;
output_name : string option;
+ print_estimate : bool;
print_source : bool;
root_choice : root_choice;
}
@@ -49,6 +51,7 @@ let parse_cmdline () let debug_overlays = ref false in
let do_copy = ref true in
let machine_readable = ref false in
+ let print_estimate = ref false in
let print_source = ref false in
let qemu_boot = ref false in
@@ -235,6 +238,8 @@ let parse_cmdline ()
s_"Set output storage location";
[ L"password-file" ], Getopt.String ("filename",
set_string_option_once "--password-file" input_password),
s_"Same as ‘-ip filename’";
+ [ L"print-estimate" ], Getopt.Set print_estimate,
+ s_"Estimate size of source and
stop";
[ L"print-source" ], Getopt.Set print_source,
s_"Print source and stop";
[ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o
qemu only)";
@@ -330,6 +335,7 @@ read the man page virt-v2v(1).
let output_options = List.rev !output_options in
let output_password = !output_password in
let output_storage = !output_storage in
+ let print_estimate = !print_estimate in
let print_source = !print_source in
let qemu_boot = !qemu_boot in
let root_choice = !root_choice in
@@ -355,6 +361,12 @@ read the man page virt-v2v(1).
exit 0
);
+ (* Some options cannot be used with --in-place. *)
+ if in_place then (
+ if print_estimate then
+ error (f_"--in-place and --print-estimate cannot be used
together")
+ );
+
(* Input transport affects whether some input options should or
* should not be used.
*)
@@ -620,8 +632,9 @@ read the man page virt-v2v(1).
output_format, output_alloc in
{
- compressed; debug_overlays; do_copy; in_place; network_map;
+ compressed; debug_overlays; do_copy; in_place;
+ machine_readable; network_map;
output_alloc; output_format; output_name;
- print_source; root_choice;
+ print_estimate; print_source; root_choice;
},
input, output
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 25beb1c95..9b5bd4098 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -23,10 +23,12 @@ type cmdline = {
debug_overlays : bool;
do_copy : bool;
in_place : bool;
+ machine_readable : bool;
network_map : Networks.t;
output_alloc : Types.output_allocation;
output_format : string option;
output_name : string option;
+ print_estimate : bool;
print_source : bool;
root_choice : Types.root_choice;
}
diff --git a/v2v/test-v2v-print-estimate.sh b/v2v/test-v2v-print-estimate.sh
new file mode 100755
index 000000000..d952d1a37
--- /dev/null
+++ b/v2v/test-v2v-print-estimate.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -
+# libguestfs virt-v2v test script
+# Copyright (C) 2018 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test --print-estimate option.
+
+set -e
+
+$TEST_FUNCTIONS
+skip_if_skipped
+skip_unless_phony_guest windows.img
+
+f=test-v2v-print-estimate.out
+rm -f $f
+
+du -s -B 1 ../test-data/phony-guests/windows.img
+
+$VG virt-v2v --debug-gc \
+ -i libvirtxml test-v2v-print-source.xml \
+ -o local -os $(pwd) \
+ --print-estimate --quiet > $f
+
+echo -n "Estimate: "
+cat $f
+
+# Check the output is a single number.
+if [ "$(cat $f | wc -l)" -ne 1 ]; then
+ echo "$0: expecting one line of output"
+ exit 1
+fi
+grep -E '^[[:digit:]]+$' $f
+
+rm -f $f
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 1775200d3..e24a9adf6 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -104,6 +104,12 @@ let rec main () (* Decrypt the disks. *)
inspect_decrypt g;
+ (* Print source disk size estimate and stop. *)
+ if cmdline.print_estimate then (
+ print_source_disk_size_estimate cmdline g;
+ exit 0
+ );
+
(* Inspection - this also mounts up the filesystems. *)
(match conversion_mode with
| Copying _ -> message (f_"Inspecting the overlay")
@@ -371,6 +377,48 @@ and print_mpstat chan { mp_dev = dev; mp_path = path;
fprintf chan " bsize=%Ld blocks=%Ld bfree=%Ld bavail=%Ld\n"
s.Guestfs.bsize s.Guestfs.blocks s.Guestfs.bfree s.Guestfs.bavail
+(* Print the estimated size of the source disk(s).
+ *
+ * These are somewhat related to mpstats above, except that
+ * we must also collect information about devices which do
+ * not contain mountable filesystems and so we must assume
+ * are copied completely.
+ *
+ * This function will unmount and mount filesystems
+ * randomly, but that's OK because we exit afterwards.
+ *)
+and print_source_disk_size_estimate cmdline g + let fses = List.map fst
(g#list_filesystems ()) in
+
+ let size + List.fold_left (
+ fun size dev ->
+ g#umount_all ();
+ let mounted + try g#mount_ro dev "/"; true
+ with G.Error _ -> false in
+
+ let sz, what + if mounted then (
+ let { Guestfs.bfree; blocks; bsize } = g#statvfs "/" in
+ (blocks -^ bfree) *^ bsize, "filesystem"
+ )
+ else (
+ (* Assume the full size of the filesystem will have
+ * to be copied.
+ *)
+ g#blockdev_getsize64 dev, "device"
+ ) in
+
+ debug "print-estimate: %s %s uses %Ld (%s)"
+ what dev sz (human_size sz);
+ size +^ sz
+ ) 0L fses in
+
+ g#umount_all ();
+
+ printf "%Ld\n" size
+
(* Conversion can fail if there is no space on the guest filesystems
* (RHBZ#1139543). To avoid this situation, check there is some
* headroom. Mainly we care about the root filesystem.
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index 303fe425c..80b765983 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -794,6 +794,23 @@ C<root>.
You will get an error if virt-v2v is unable to mount/write to the
Export Storage Domain.
+=item B<--print-estimate>
+
+Print the estimated size of the data which will be copied from the
+source disk(s) and stop.
+
+A single number is printed on stdout which is the estimated size of
+data that will be copied, in bytes.
+
+This estimate is the sum across all disks, because guest features such
+as LVM and MD means that it is not meaningful to provide separate
+estimates for each disk. The size does not include deleted files and
+empty space in the source, but it cannot detect used-but-zero space
+and so it usually overestimates.
+
+You usually want to use this option in conjunction with the I<--quiet>
+option so that the result is not mixed in with standard messages.
+
=item B<--print-source>
Print information about the source guest and stop. This option is
--
2.18.0
Nir Soffer
2018-Aug-14 18:31 UTC
Re: [Libguestfs] [PATCH] v2v: Add --print-estimate option to print source size estimate.
On Tue, Aug 14, 2018 at 8:29 PM Richard W.M. Jones <rjones@redhat.com> wrote:> This option prints the estimated size of the data that will be copied > from the source disk. > > For interest, the test prints: > > 3747840 ../test-data/phony-guests/windows.img > Estimate: 3710976 >Why not use qemu-img measure on the overlay? It gives a conservative estimate that will never fail, based on the allocated blocks and additional metadata required for the destination file format. Nir> --- > v2v/Makefile.am | 2 ++ > v2v/cmdline.ml | 17 ++++++++++-- > v2v/cmdline.mli | 2 ++ > v2v/test-v2v-print-estimate.sh | 47 +++++++++++++++++++++++++++++++++ > v2v/v2v.ml | 48 ++++++++++++++++++++++++++++++++++ > v2v/virt-v2v.pod | 17 ++++++++++++ > 6 files changed, 131 insertions(+), 2 deletions(-) > > diff --git a/v2v/Makefile.am b/v2v/Makefile.am > index 5461055d1..2ea0dd1d5 100644 > --- a/v2v/Makefile.am > +++ b/v2v/Makefile.am > @@ -379,6 +379,7 @@ TESTS += \ > test-v2v-oa-option.sh \ > test-v2v-of-option.sh \ > test-v2v-on-option.sh \ > + test-v2v-print-estimate.sh \ > test-v2v-print-source.sh \ > test-v2v-sound.sh \ > $(SLOW_TESTS) \ > @@ -534,6 +535,7 @@ EXTRA_DIST += \ > test-v2v-oa-option.sh \ > test-v2v-of-option.sh \ > test-v2v-on-option.sh \ > + test-v2v-print-estimate.sh \ > test-v2v-print-source.expected \ > test-v2v-print-source.sh \ > test-v2v-print-source.xml \ > diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml > index 5b2df3555..74cc27714 100644 > --- a/v2v/cmdline.ml > +++ b/v2v/cmdline.ml > @@ -33,10 +33,12 @@ type cmdline = { > debug_overlays : bool; > do_copy : bool; > in_place : bool; > + machine_readable : bool; > network_map : Networks.t; > output_alloc : output_allocation; > output_format : string option; > output_name : string option; > + print_estimate : bool; > print_source : bool; > root_choice : root_choice; > } > @@ -49,6 +51,7 @@ let parse_cmdline () > let debug_overlays = ref false in > let do_copy = ref true in > let machine_readable = ref false in > + let print_estimate = ref false in > let print_source = ref false in > let qemu_boot = ref false in > > @@ -235,6 +238,8 @@ let parse_cmdline () > s_"Set output storage location"; > [ L"password-file" ], Getopt.String ("filename", > set_string_option_once "--password-file" input_password), > s_"Same as ‘-ip filename’"; > + [ L"print-estimate" ], Getopt.Set print_estimate, > + s_"Estimate size of source and stop"; > [ L"print-source" ], Getopt.Set print_source, > s_"Print source and stop"; > [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o qemu > only)"; > @@ -330,6 +335,7 @@ read the man page virt-v2v(1). > let output_options = List.rev !output_options in > let output_password = !output_password in > let output_storage = !output_storage in > + let print_estimate = !print_estimate in > let print_source = !print_source in > let qemu_boot = !qemu_boot in > let root_choice = !root_choice in > @@ -355,6 +361,12 @@ read the man page virt-v2v(1). > exit 0 > ); > > + (* Some options cannot be used with --in-place. *) > + if in_place then ( > + if print_estimate then > + error (f_"--in-place and --print-estimate cannot be used together") > + ); > + > (* Input transport affects whether some input options should or > * should not be used. > *) > @@ -620,8 +632,9 @@ read the man page virt-v2v(1). > output_format, output_alloc in > > { > - compressed; debug_overlays; do_copy; in_place; network_map; > + compressed; debug_overlays; do_copy; in_place; > + machine_readable; network_map; > output_alloc; output_format; output_name; > - print_source; root_choice; > + print_estimate; print_source; root_choice; > }, > input, output > diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli > index 25beb1c95..9b5bd4098 100644 > --- a/v2v/cmdline.mli > +++ b/v2v/cmdline.mli > @@ -23,10 +23,12 @@ type cmdline = { > debug_overlays : bool; > do_copy : bool; > in_place : bool; > + machine_readable : bool; > network_map : Networks.t; > output_alloc : Types.output_allocation; > output_format : string option; > output_name : string option; > + print_estimate : bool; > print_source : bool; > root_choice : Types.root_choice; > } > diff --git a/v2v/test-v2v-print-estimate.sh > b/v2v/test-v2v-print-estimate.sh > new file mode 100755 > index 000000000..d952d1a37 > --- /dev/null > +++ b/v2v/test-v2v-print-estimate.sh > @@ -0,0 +1,47 @@ > +#!/bin/bash - > +# libguestfs virt-v2v test script > +# Copyright (C) 2018 Red Hat Inc. > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; either version 2 of the License, or > +# (at your option) any later version. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street > <https://maps.google.com/?q=51+Franklin+Street&entry=gmail&source=g>, > Fifth Floor, Boston, MA 02110-1301 USA. > + > +# Test --print-estimate option. > + > +set -e > + > +$TEST_FUNCTIONS > +skip_if_skipped > +skip_unless_phony_guest windows.img > + > +f=test-v2v-print-estimate.out > +rm -f $f > + > +du -s -B 1 ../test-data/phony-guests/windows.img > + > +$VG virt-v2v --debug-gc \ > + -i libvirtxml test-v2v-print-source.xml \ > + -o local -os $(pwd) \ > + --print-estimate --quiet > $f > + > +echo -n "Estimate: " > +cat $f > + > +# Check the output is a single number. > +if [ "$(cat $f | wc -l)" -ne 1 ]; then > + echo "$0: expecting one line of output" > + exit 1 > +fi > +grep -E '^[[:digit:]]+$' $f > + > +rm -f $f > diff --git a/v2v/v2v.ml b/v2v/v2v.ml > index 1775200d3..e24a9adf6 100644 > --- a/v2v/v2v.ml > +++ b/v2v/v2v.ml > @@ -104,6 +104,12 @@ let rec main () > (* Decrypt the disks. *) > inspect_decrypt g; > > + (* Print source disk size estimate and stop. *) > + if cmdline.print_estimate then ( > + print_source_disk_size_estimate cmdline g; > + exit 0 > + ); > + > (* Inspection - this also mounts up the filesystems. *) > (match conversion_mode with > | Copying _ -> message (f_"Inspecting the overlay") > @@ -371,6 +377,48 @@ and print_mpstat chan { mp_dev = dev; mp_path = path; > fprintf chan " bsize=%Ld blocks=%Ld bfree=%Ld bavail=%Ld\n" > s.Guestfs.bsize s.Guestfs.blocks s.Guestfs.bfree s.Guestfs.bavail > > +(* Print the estimated size of the source disk(s). > + * > + * These are somewhat related to mpstats above, except that > + * we must also collect information about devices which do > + * not contain mountable filesystems and so we must assume > + * are copied completely. > + * > + * This function will unmount and mount filesystems > + * randomly, but that's OK because we exit afterwards. > + *) > +and print_source_disk_size_estimate cmdline g > + let fses = List.map fst (g#list_filesystems ()) in > + > + let size > + List.fold_left ( > + fun size dev -> > + g#umount_all (); > + let mounted > + try g#mount_ro dev "/"; true > + with G.Error _ -> false in > + > + let sz, what > + if mounted then ( > + let { Guestfs.bfree; blocks; bsize } = g#statvfs "/" in > + (blocks -^ bfree) *^ bsize, "filesystem" > + ) > + else ( > + (* Assume the full size of the filesystem will have > + * to be copied. > + *) > + g#blockdev_getsize64 dev, "device" > + ) in > + > + debug "print-estimate: %s %s uses %Ld (%s)" > + what dev sz (human_size sz); > + size +^ sz > + ) 0L fses in > + > + g#umount_all (); > + > + printf "%Ld\n" size > + > (* Conversion can fail if there is no space on the guest filesystems > * (RHBZ#1139543). To avoid this situation, check there is some > * headroom. Mainly we care about the root filesystem. > diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod > index 303fe425c..80b765983 100644 > --- a/v2v/virt-v2v.pod > +++ b/v2v/virt-v2v.pod > @@ -794,6 +794,23 @@ C<root>. > You will get an error if virt-v2v is unable to mount/write to the > Export Storage Domain. > > +=item B<--print-estimate> > + > +Print the estimated size of the data which will be copied from the > +source disk(s) and stop. > + > +A single number is printed on stdout which is the estimated size of > +data that will be copied, in bytes. > + > +This estimate is the sum across all disks, because guest features such > +as LVM and MD means that it is not meaningful to provide separate > +estimates for each disk. The size does not include deleted files and > +empty space in the source, but it cannot detect used-but-zero space > +and so it usually overestimates. > + > +You usually want to use this option in conjunction with the I<--quiet> > +option so that the result is not mixed in with standard messages. > + > =item B<--print-source> > > Print information about the source guest and stop. This option is > -- > 2.18.0> _______________________________________________ > Libguestfs mailing list > Libguestfs@redhat.com > https://www.redhat.com/mailman/listinfo/libguestfs
Richard W.M. Jones
2018-Aug-15 10:26 UTC
Re: [Libguestfs] [PATCH] v2v: Add --print-estimate option to print source size estimate.
On Tue, Aug 14, 2018 at 09:31:06PM +0300, Nir Soffer wrote:> On Tue, Aug 14, 2018 at 8:29 PM Richard W.M. Jones <rjones@redhat.com> > wrote: > > > This option prints the estimated size of the data that will be copied > > from the source disk. > > > > For interest, the test prints: > > > > 3747840 ../test-data/phony-guests/windows.img > > Estimate: 3710976 > > > > Why not use qemu-img measure on the overlay?Yes this would be better, but oddly it doesn't work properly when the output format is set to 'raw': /usr/bin/qemu-img 'measure' '-f' 'qcow2' '-O' 'raw' '--output=human' '/home/rjones/d/libguestfs/tmp/v2vovl62b7c2.qcow2' required size: 6442450944 fully allocated size: 6442450944 However it's OK if the output format is set to 'qcow2': /usr/bin/qemu-img 'measure' '-f' 'qcow2' '-O' 'qcow2' '--output=human' '/home/rjones/d/libguestfs/tmp/v2vovla53d7c.qcow2' required size: 1047986176 fully allocated size: 6443696128 I guess it ignores sparseness of raw images, but I can't see a way to specify that on the command line. OTOH the qcow2 figure is probably a good enough guess for our purposes (ie. estimating how much data will be copied). Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-p2v converts physical machines to virtual machines. Boot with a live CD or over the network (PXE) and turn machines into KVM guests. http://libguestfs.org/virt-v2v
Reasonably Related Threads
- Re: [PATCH] v2v: Add --print-estimate option to print source size estimate.
- [PATCH v2] v2v: Add --print-estimate option to print copy size estimate.
- Re: [PATCH 2/2] v2v: Add --print-target to display overlay and target information.
- [PATCH v6] v2v: Add --print-estimate option to print copy size estimate.
- [PATCH v4] v2v: Add --print-estimate option to print copy size estimate.