Libguestfs - Aug 2018 - [PATCH v6] v2v: Add --print-estimate option to print copy size estimate.

v6:

 - Make the text output a bit nicer.

 - Changes as suggested to Measure_disk module temp file & json parsing.

 - Use jq to test JSON output.

 - Retest.

This option prints the estimated size of the data that will be copied
from the source disk.

Currently this overestimates by the size of the qcow2 header, but for
real disk images that doesn't matter much.

For example:

$ virt-builder fedora-27
$ virt-v2v -i disk fedora-27.img -o null --print-estimate
[...]
virt-v2v: This guest has virtio drivers installed.
[  44.0] Mapping filesystem data to avoid copying unused and blank areas
[  44.5] Closing the overlay
disk 1: 1047920640
total: 1047920640
---
 v2v/Makefile.am                |  5 +++
 v2v/cmdline.ml                 | 13 +++++++-
 v2v/cmdline.mli                |  1 +
 v2v/measure_disk.ml            | 56 ++++++++++++++++++++++++++++++++++
 v2v/measure_disk.mli           | 21 +++++++++++++
 v2v/test-v2v-print-estimate.sh | 46 ++++++++++++++++++++++++++++
 v2v/v2v.ml                     | 29 ++++++++++++++++--
 v2v/virt-v2v.pod               | 31 +++++++++++++++++++
 8 files changed, 199 insertions(+), 3 deletions(-)

diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 97dd44ec9..e1fee77b0 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -63,6 +63,7 @@ SOURCES_MLI = \
 	linux.mli \
 	linux_bootloaders.mli \
 	linux_kernels.mli \
+	measure_disk.mli \
 	modules_list.mli \
 	name_from_disk.mli \
 	networks.mli \
@@ -139,6 +140,7 @@ SOURCES_ML = \
 	output_vdsm.ml \
 	inspect_source.ml \
 	target_bus_assignment.ml \
+	measure_disk.ml \
 	networks.ml \
 	cmdline.ml \
 	v2v.ml
@@ -203,6 +205,7 @@ OCAMLCLIBS = \
 	-lqemuopts \
 	$(LIBVIRT_LIBS) \
 	$(LIBXML2_LIBS) \
+	$(JANSSON_LIBS) \
 	$(LIBINTL) \
 	-lgnu
 
@@ -378,6 +381,7 @@ TESTS += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.sh \
 	test-v2v-sound.sh \
 	$(SLOW_TESTS) \
@@ -533,6 +537,7 @@ EXTRA_DIST += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.expected \
 	test-v2v-print-source.sh \
 	test-v2v-print-source.xml \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index c61d83f66..a5c125361 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -37,6 +37,7 @@ type cmdline = {
   output_alloc : output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : root_choice;
 }
@@ -48,6 +49,7 @@ let parse_cmdline ()    let compressed = ref false in
   let debug_overlays = ref false in
   let do_copy = ref true in
+  let print_estimate = ref false in
   let print_source = ref false in
   let qemu_boot = ref false in
 
@@ -232,6 +234,8 @@ let parse_cmdline ()                                     
s_"Set output storage location";
     [ L"password-file" ], Getopt.String ("filename",
set_string_option_once "--password-file" input_password),
                                     s_"Same as ‘-ip filename’";
+    [ L"print-estimate" ], Getopt.Set print_estimate,
+                                    s_"Estimate size of source and
stop";
     [ L"print-source" ], Getopt.Set print_source,
                                     s_"Print source and stop";
     [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o
qemu only)";
@@ -326,6 +330,7 @@ read the man page virt-v2v(1).
   let output_options = List.rev !output_options in
   let output_password = !output_password in
   let output_storage = !output_storage in
+  let print_estimate = !print_estimate in
   let print_source = !print_source in
   let qemu_boot = !qemu_boot in
   let root_choice = !root_choice in
@@ -353,6 +358,12 @@ read the man page virt-v2v(1).
   | _, _ -> ()
   );
 
+  (* Some options cannot be used with --in-place. *)
+  if in_place then (
+    if print_estimate then
+      error (f_"--in-place and --print-estimate cannot be used
together")
+  );
+
   (* Input transport affects whether some input options should or
    * should not be used.
    *)
@@ -620,6 +631,6 @@ read the man page virt-v2v(1).
   {
     compressed; debug_overlays; do_copy; in_place; network_map;
     output_alloc; output_format; output_name;
-    print_source; root_choice;
+    print_estimate; print_source; root_choice;
   },
   input, output
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 25beb1c95..de6281bab 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -27,6 +27,7 @@ type cmdline = {
   output_alloc : Types.output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : Types.root_choice;
 }
diff --git a/v2v/measure_disk.ml b/v2v/measure_disk.ml
new file mode 100644
index 000000000..327b6fc8d
--- /dev/null
+++ b/v2v/measure_disk.ml
@@ -0,0 +1,56 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open JSON_parser
+open Common_gettext.Gettext
+
+(* Run qemu-img measure on a disk. *)
+
+let measure ?format filename +  let cmd = ref [] in
+  List.push_back_list cmd [Guestfs_config.qemu_img; "measure"];
+  (match format with
+   | None -> ()
+   | Some format ->
+      List.push_back_list cmd ["-f"; format]
+  );
+  (* For use of -O qcow2 here, see this thread:
+   * https://www.redhat.com/archives/libguestfs/2018-August/thread.html#00142
+   *)
+  List.push_back_list cmd ["-O"; "qcow2"];
+  List.push_back cmd "--output=json";
+  List.push_back cmd filename;
+
+  let json, chan = Filename.open_temp_file "v2vmeasure"
".json" in
+  unlink_on_exit json;
+  let fd = Unix.descr_of_out_channel chan in
+  if run_command ~stdout_fd:fd !cmd <> 0 then
+    error (f_"qemu-img measure failed, see earlier errors");
+  (* Note that run_command closes fd. *)
+
+  let json = json_parser_tree_parse_file json in
+  debug "qemu-img measure output parsed as: %s"
+        (JSON.string_of_doc ~fmt:JSON.Indented ["", json]);
+
+  (* We're expecting the tree to contain nodes:
+   * Dict [ "required", Int number; "fully-allocated", Int
number ]
+   * Of course the array could appear in any order.
+   *)
+  object_get_number "required" json
diff --git a/v2v/measure_disk.mli b/v2v/measure_disk.mli
new file mode 100644
index 000000000..efaa76105
--- /dev/null
+++ b/v2v/measure_disk.mli
@@ -0,0 +1,21 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Run qemu-img measure on a disk. *)
+
+val measure : ?format:string -> string -> int64
diff --git a/v2v/test-v2v-print-estimate.sh b/v2v/test-v2v-print-estimate.sh
new file mode 100755
index 000000000..a50e9a35e
--- /dev/null
+++ b/v2v/test-v2v-print-estimate.sh
@@ -0,0 +1,46 @@
+#!/bin/bash -
+# libguestfs virt-v2v test script
+# Copyright (C) 2018 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test --print-estimate option.
+
+set -e
+
+$TEST_FUNCTIONS
+skip_if_skipped
+skip_unless_phony_guest windows.img
+skip_unless jq --version
+
+f=test-v2v-print-estimate.out
+rm -f $f
+
+echo "Actual:"
+du -s -B 1 ../test-data/phony-guests/windows.img
+
+$VG virt-v2v --debug-gc \
+    -i libvirtxml test-v2v-print-source.xml \
+    -o local -os $(pwd) \
+    --print-estimate \
+    --machine-readable=file:$f
+
+echo -n "Estimate: "
+cat $f
+
+# Check the total field exists and is numeric.
+jq '.total' $f | grep -Esq '^[[:digit:]]+$'
+
+rm -f $f
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 1775200d3..53a1041c5 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -48,6 +48,8 @@ type mpstat = {
 
 let () = Random.self_init ()
 
+let sum = List.fold_left (+^) 0L
+
 let rec main ()    (* Handle the command line. *)
   let cmdline, input, output = parse_cmdline () in
@@ -156,6 +158,12 @@ let rec main ()    (match conversion_mode with
    | In_place -> ()
    | Copying overlays ->
+      (* Print copy size estimate and stop. *)
+      if cmdline.print_estimate then (
+        print_estimate overlays;
+        exit 0
+      );
+
       message (f_"Assigning disks to buses");
       let target_buses          Target_bus_assignment.target_bus_assignment
source guestcaps in
@@ -487,8 +495,6 @@ and do_fstrim g inspect   *     sdb final estimate size = 3
- (3*1.35/4) = 1.9875 GB
  *)
 and estimate_target_size mpstats overlays -  let sum = List.fold_left (+^) 0L
in
-
   (* (1) *)
   let fs_total_size      sum (
@@ -617,6 +623,25 @@ and get_target_formats cmdline output overlays       
format
   ) overlays
 
+and print_estimate overlays +  let estimates +    List.map (
+      fun { ov_overlay_file } ->
+        Measure_disk.measure ~format:"qcow2" ov_overlay_file
+    ) overlays in
+  let total = sum estimates in
+
+  match machine_readable () with
+  | None ->
+     List.iteri (fun i e -> printf "disk %d: %Ld\n" (i+1) e)
estimates;
+     printf "total: %Ld\n" total
+  | Some {pr} ->
+     let json = [
+       "disks", JSON.List (List.map (fun i -> JSON.Int i)
estimates);
+       "total", JSON.Int total
+     ] in
+     pr "%s\n" (JSON.string_of_doc ~fmt:JSON.Indented json)
+
 (* Does the guest require UEFI on the target? *)
 and get_target_firmware inspect guestcaps source output    message
(f_"Checking if the guest needs BIOS or UEFI to boot");
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index fa7836659..c6810aab7 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -796,6 +796,37 @@ C<root>.
 You will get an error if virt-v2v is unable to mount/write to the
 Export Storage Domain.
 
+=item B<--print-estimate>
+
+Print the estimated size of the data which will be copied from the
+source disk(s) and stop.  One number (the size in bytes) is printed
+per disk, and a total:
+
+ $ virt-v2v --print-estimate
+ ...
+ disk 1: 100000
+ disk 2: 200000
+ total: 300000
+
+With the I<--machine-readable> option you get JSON output which can be
+directed into a file or elsewhere:
+
+ $ virt-v2v --print-estimate --machine-readable=file:estimates
+ ...
+ $ cat estimates
+ {
+  "disks": [ 100000, 200000 ],
+  "total": 300000
+ }
+
+When using this option you must specify an output mode.  This is
+because virt-v2v has to perform the conversion in order to print the
+estimate, and the conversion depends on the output mode.  Using
+I<-o null> should be safe for most purposes.
+
+When this option is used along with I<--machine-readable> you can
+direct the output to an alternate file.
+
 =item B<--print-source>
 
 Print information about the source guest and stop.  This option is
-- 
2.18.0

On Friday, 24 August 2018 13:24:14 CEST Richard W.M. Jones
wrote:
> This option prints the estimated size of the data that will be copied
> from the source disk.
> 
> Currently this overestimates by the size of the qcow2 header, but for
> real disk images that doesn't matter much.
> 
> For example:
> 
> $ virt-builder fedora-27
> $ virt-v2v -i disk fedora-27.img -o null --print-estimate
> [...]
> virt-v2v: This guest has virtio drivers installed.
> [  44.0] Mapping filesystem data to avoid copying unused and blank areas
> [  44.5] Closing the overlay
> disk 1: 1047920640
> total: 1047920640
> ---
The non-machine readable output could be better, but I guess I will send
a patch for it later ...

LGTM otherwise.

-- 
Pino Toscano