Libguestfs - Aug 2018 - [PATCH v4] v2v: Add --print-estimate option to print copy size estimate.

v4:

- Same as v3, but depends on and uses new --machine-readable work.

Rich.

This option prints the estimated size of the data that will be copied
from the source disk.

Currently this overestimates by the size of the qcow2 header, but for
real disk images that doesn't matter much.

For example:

$ virt-builder fedora-27
$ virt-v2v -i disk fedora-27.img -o null --print-estimate
[...]
virt-v2v: This guest has virtio drivers installed.
[  44.0] Mapping filesystem data to avoid copying unused and blank areas
[  44.5] Closing the overlay
1047920640
---
 v2v/Makefile.am                |  5 +++
 v2v/cmdline.ml                 | 13 +++++++-
 v2v/cmdline.mli                |  1 +
 v2v/measure_disk.ml            | 59 ++++++++++++++++++++++++++++++++++
 v2v/measure_disk.mli           | 21 ++++++++++++
 v2v/test-v2v-print-estimate.sh | 44 +++++++++++++++++++++++++
 v2v/v2v.ml                     | 20 ++++++++++--
 v2v/virt-v2v.pod               | 14 ++++++++
 8 files changed, 174 insertions(+), 3 deletions(-)

diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 97dd44ec9..e1fee77b0 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -63,6 +63,7 @@ SOURCES_MLI = \
 	linux.mli \
 	linux_bootloaders.mli \
 	linux_kernels.mli \
+	measure_disk.mli \
 	modules_list.mli \
 	name_from_disk.mli \
 	networks.mli \
@@ -139,6 +140,7 @@ SOURCES_ML = \
 	output_vdsm.ml \
 	inspect_source.ml \
 	target_bus_assignment.ml \
+	measure_disk.ml \
 	networks.ml \
 	cmdline.ml \
 	v2v.ml
@@ -203,6 +205,7 @@ OCAMLCLIBS = \
 	-lqemuopts \
 	$(LIBVIRT_LIBS) \
 	$(LIBXML2_LIBS) \
+	$(JANSSON_LIBS) \
 	$(LIBINTL) \
 	-lgnu
 
@@ -378,6 +381,7 @@ TESTS += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.sh \
 	test-v2v-sound.sh \
 	$(SLOW_TESTS) \
@@ -533,6 +537,7 @@ EXTRA_DIST += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.expected \
 	test-v2v-print-source.sh \
 	test-v2v-print-source.xml \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 10cbb90e6..da525de44 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -37,6 +37,7 @@ type cmdline = {
   output_alloc : output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : root_choice;
 }
@@ -48,6 +49,7 @@ let parse_cmdline ()    let compressed = ref false in
   let debug_overlays = ref false in
   let do_copy = ref true in
+  let print_estimate = ref false in
   let print_source = ref false in
   let qemu_boot = ref false in
 
@@ -232,6 +234,8 @@ let parse_cmdline ()                                     
s_"Set output storage location";
     [ L"password-file" ], Getopt.String ("filename",
set_string_option_once "--password-file" input_password),
                                     s_"Same as ‘-ip filename’";
+    [ L"print-estimate" ], Getopt.Set print_estimate,
+                                    s_"Estimate size of source and
stop";
     [ L"print-source" ], Getopt.Set print_source,
                                     s_"Print source and stop";
     [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o
qemu only)";
@@ -326,6 +330,7 @@ read the man page virt-v2v(1).
   let output_options = List.rev !output_options in
   let output_password = !output_password in
   let output_storage = !output_storage in
+  let print_estimate = !print_estimate in
   let print_source = !print_source in
   let qemu_boot = !qemu_boot in
   let root_choice = !root_choice in
@@ -351,6 +356,12 @@ read the man page virt-v2v(1).
     exit 0
   );
 
+  (* Some options cannot be used with --in-place. *)
+  if in_place then (
+    if print_estimate then
+      error (f_"--in-place and --print-estimate cannot be used
together")
+  );
+
   (* Input transport affects whether some input options should or
    * should not be used.
    *)
@@ -618,6 +629,6 @@ read the man page virt-v2v(1).
   {
     compressed; debug_overlays; do_copy; in_place; network_map;
     output_alloc; output_format; output_name;
-    print_source; root_choice;
+    print_estimate; print_source; root_choice;
   },
   input, output
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 25beb1c95..de6281bab 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -27,6 +27,7 @@ type cmdline = {
   output_alloc : Types.output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : Types.root_choice;
 }
diff --git a/v2v/measure_disk.ml b/v2v/measure_disk.ml
new file mode 100644
index 000000000..497f2a72e
--- /dev/null
+++ b/v2v/measure_disk.ml
@@ -0,0 +1,59 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open JSON_parser
+open Common_gettext.Gettext
+
+(* Run qemu-img measure on a disk. *)
+
+let measure ?format filename +  let cmd = ref [] in
+  List.push_back_list cmd [Guestfs_config.qemu_img; "measure"];
+  (match format with
+   | None -> ()
+   | Some format ->
+      List.push_back_list cmd ["-f"; format]
+  );
+  (* For use of -O qcow2 here, see this thread:
+   * https://www.redhat.com/archives/libguestfs/2018-August/thread.html#00142
+   *)
+  List.push_back_list cmd ["-O"; "qcow2"];
+  List.push_back cmd "--output=json";
+  List.push_back cmd filename;
+
+  let json = Filename.temp_file "v2vmeasure" ".json" in
+  unlink_on_exit json;
+
+  let fd = Unix.openfile json [O_WRONLY; O_CREAT; O_TRUNC] 0o600 in
+  if run_command ~stdout_fd:fd !cmd <> 0 then
+    error (f_"qemu-img measure failed, see earlier errors");
+  (* Note that run_command closes fd. *)
+
+  let json = read_whole_file json in
+  debug "qemu-img measure output: %s" json;
+  let json = json_parser_tree_parse json in
+  debug "qemu-img measure output parsed as: %s"
+        (JSON.string_of_doc ~fmt:JSON.Indented ["", json]);
+
+  (* We're expecting the tree to contain nodes:
+   * Dict [ "required", Int number; "fully-allocated", Int
number ]
+   * Of course the array could appear in any order.
+   *)
+  object_get_number "required" json
diff --git a/v2v/measure_disk.mli b/v2v/measure_disk.mli
new file mode 100644
index 000000000..efaa76105
--- /dev/null
+++ b/v2v/measure_disk.mli
@@ -0,0 +1,21 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Run qemu-img measure on a disk. *)
+
+val measure : ?format:string -> string -> int64
diff --git a/v2v/test-v2v-print-estimate.sh b/v2v/test-v2v-print-estimate.sh
new file mode 100755
index 000000000..10388f896
--- /dev/null
+++ b/v2v/test-v2v-print-estimate.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -
+# libguestfs virt-v2v test script
+# Copyright (C) 2018 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test --print-estimate option.
+
+set -e
+
+$TEST_FUNCTIONS
+skip_if_skipped
+skip_unless_phony_guest windows.img
+
+f=test-v2v-print-estimate.out
+rm -f $f
+
+echo "Actual:"
+du -s -B 1 ../test-data/phony-guests/windows.img
+
+$VG virt-v2v --debug-gc \
+    -i libvirtxml test-v2v-print-source.xml \
+    -o local -os $(pwd) \
+    --print-estimate --quiet > $f
+
+echo -n "Estimate: "
+cat $f
+
+# Check the output looks reasonable.
+grep -E '^[[:digit:]]+$' $f
+
+rm -f $f
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 1775200d3..d7b791885 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -48,6 +48,8 @@ type mpstat = {
 
 let () = Random.self_init ()
 
+let sum = List.fold_left (+^) 0L
+
 let rec main ()    (* Handle the command line. *)
   let cmdline, input, output = parse_cmdline () in
@@ -156,6 +158,22 @@ let rec main ()    (match conversion_mode with
    | In_place -> ()
    | Copying overlays ->
+      (* Print copy size estimate and stop. *)
+      if cmdline.print_estimate then (
+        let estimated_size +          sum (List.map
+                 (fun { ov_overlay_file } ->
+                   Measure_disk.measure ~format:"qcow2"
ov_overlay_file)
+                 overlays) in
+        (match machine_readable () with
+         | None ->
+            printf "%Ld\n" estimated_size
+         | Some {pr} ->
+            pr "%Ld\n" estimated_size
+        );
+        exit 0
+      );
+
       message (f_"Assigning disks to buses");
       let target_buses          Target_bus_assignment.target_bus_assignment
source guestcaps in
@@ -487,8 +505,6 @@ and do_fstrim g inspect   *     sdb final estimate size = 3
- (3*1.35/4) = 1.9875 GB
  *)
 and estimate_target_size mpstats overlays -  let sum = List.fold_left (+^) 0L
in
-
   (* (1) *)
   let fs_total_size      sum (
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index e6f08d1e6..a92ae9f88 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -794,6 +794,20 @@ C<root>.
 You will get an error if virt-v2v is unable to mount/write to the
 Export Storage Domain.
 
+=item B<--print-estimate>
+
+Print the estimated size of the data which will be copied from the
+source disk(s) and stop.  The output is a single number, which is the
+number of bytes (estimated) that virt-v2v would copy.
+
+When using this option you must specify an output mode.  This is
+because virt-v2v has to perform the conversion in order to print the
+estimate, and the conversion depends on the output mode.  Using
+I<-o null> should be safe for most purposes.
+
+When this option is used along with I<--machine-readable> you can
+direct the output to an alternate file.
+
 =item B<--print-source>
 
 Print information about the source guest and stop.  This option is
-- 
2.18.0

On Thursday, 23 August 2018 17:59:36 CEST Richard W.M. Jones
wrote:
> This option prints the estimated size of the data that will be copied
> from the source disk.
> 
> Currently this overestimates by the size of the qcow2 header, but for
> real disk images that doesn't matter much.
> 
> For example:
> 
> $ virt-builder fedora-27
> $ virt-v2v -i disk fedora-27.img -o null --print-estimate
> [...]
> virt-v2v: This guest has virtio drivers installed.
> [  44.0] Mapping filesystem data to avoid copying unused and blank areas
> [  44.5] Closing the overlay
> 1047920640
IMHO a better output is similar to what the --print-source prints --
so for example something like:

Estimation of the disks of the guest:
     Total size: 100000
         disk 1: 50000
         disk 2: 50000

While the --machine-readable a JSON snippet:

{
  "total": 100000,
  "disks": [
    50000,
    50000
  ]
}

IMHO providing the sizes of all the disks is a good idea, so it is
possible to do better estimation on the progress, in case.  Imagine
if a VM has a much bigger disk than the others, for example.
Also, I'd do the same no matter the number of disks, for coherency.
> +  let json = Filename.temp_file "v2vmeasure" ".json"
in
> +  unlink_on_exit json;
> +
> +  let fd = Unix.openfile json [O_WRONLY; O_CREAT; O_TRUNC] 0o600 in
IMHO these can be replaced by Filename.open_temp_file +
Unix.descr_of_in_channel (still with unlink_on_exit).

-- 
Pino Toscano