Libguestfs - Aug 2018 - [PATCH v5] v2v: Add --print-estimate option to print copy size estimate.

v5:

 - Normal output modified approx as suggested in previous email.

 - Machine readable output uses JSON.

This option prints the estimated size of the data that will be copied
from the source disk.

Currently this overestimates by the size of the qcow2 header, but for
real disk images that doesn't matter much.

For example:

$ virt-builder fedora-27
$ virt-v2v -i disk fedora-27.img -o null --print-estimate
[...]
virt-v2v: This guest has virtio drivers installed.
[  44.0] Mapping filesystem data to avoid copying unused and blank areas
[  44.5] Closing the overlay
1 1047920640
total 1047920640
---
 v2v/Makefile.am                |  5 +++
 v2v/cmdline.ml                 | 13 +++++++-
 v2v/cmdline.mli                |  1 +
 v2v/measure_disk.ml            | 59 ++++++++++++++++++++++++++++++++++
 v2v/measure_disk.mli           | 21 ++++++++++++
 v2v/test-v2v-print-estimate.sh | 45 ++++++++++++++++++++++++++
 v2v/v2v.ml                     | 29 +++++++++++++++--
 v2v/virt-v2v.pod               | 31 ++++++++++++++++++
 8 files changed, 201 insertions(+), 3 deletions(-)

diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 97dd44ec9..e1fee77b0 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -63,6 +63,7 @@ SOURCES_MLI = \
 	linux.mli \
 	linux_bootloaders.mli \
 	linux_kernels.mli \
+	measure_disk.mli \
 	modules_list.mli \
 	name_from_disk.mli \
 	networks.mli \
@@ -139,6 +140,7 @@ SOURCES_ML = \
 	output_vdsm.ml \
 	inspect_source.ml \
 	target_bus_assignment.ml \
+	measure_disk.ml \
 	networks.ml \
 	cmdline.ml \
 	v2v.ml
@@ -203,6 +205,7 @@ OCAMLCLIBS = \
 	-lqemuopts \
 	$(LIBVIRT_LIBS) \
 	$(LIBXML2_LIBS) \
+	$(JANSSON_LIBS) \
 	$(LIBINTL) \
 	-lgnu
 
@@ -378,6 +381,7 @@ TESTS += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.sh \
 	test-v2v-sound.sh \
 	$(SLOW_TESTS) \
@@ -533,6 +537,7 @@ EXTRA_DIST += \
 	test-v2v-oa-option.sh \
 	test-v2v-of-option.sh \
 	test-v2v-on-option.sh \
+	test-v2v-print-estimate.sh \
 	test-v2v-print-source.expected \
 	test-v2v-print-source.sh \
 	test-v2v-print-source.xml \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index c61d83f66..a5c125361 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -37,6 +37,7 @@ type cmdline = {
   output_alloc : output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : root_choice;
 }
@@ -48,6 +49,7 @@ let parse_cmdline ()    let compressed = ref false in
   let debug_overlays = ref false in
   let do_copy = ref true in
+  let print_estimate = ref false in
   let print_source = ref false in
   let qemu_boot = ref false in
 
@@ -232,6 +234,8 @@ let parse_cmdline ()                                     
s_"Set output storage location";
     [ L"password-file" ], Getopt.String ("filename",
set_string_option_once "--password-file" input_password),
                                     s_"Same as ‘-ip filename’";
+    [ L"print-estimate" ], Getopt.Set print_estimate,
+                                    s_"Estimate size of source and
stop";
     [ L"print-source" ], Getopt.Set print_source,
                                     s_"Print source and stop";
     [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o
qemu only)";
@@ -326,6 +330,7 @@ read the man page virt-v2v(1).
   let output_options = List.rev !output_options in
   let output_password = !output_password in
   let output_storage = !output_storage in
+  let print_estimate = !print_estimate in
   let print_source = !print_source in
   let qemu_boot = !qemu_boot in
   let root_choice = !root_choice in
@@ -353,6 +358,12 @@ read the man page virt-v2v(1).
   | _, _ -> ()
   );
 
+  (* Some options cannot be used with --in-place. *)
+  if in_place then (
+    if print_estimate then
+      error (f_"--in-place and --print-estimate cannot be used
together")
+  );
+
   (* Input transport affects whether some input options should or
    * should not be used.
    *)
@@ -620,6 +631,6 @@ read the man page virt-v2v(1).
   {
     compressed; debug_overlays; do_copy; in_place; network_map;
     output_alloc; output_format; output_name;
-    print_source; root_choice;
+    print_estimate; print_source; root_choice;
   },
   input, output
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 25beb1c95..de6281bab 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -27,6 +27,7 @@ type cmdline = {
   output_alloc : Types.output_allocation;
   output_format : string option;
   output_name : string option;
+  print_estimate : bool;
   print_source : bool;
   root_choice : Types.root_choice;
 }
diff --git a/v2v/measure_disk.ml b/v2v/measure_disk.ml
new file mode 100644
index 000000000..497f2a72e
--- /dev/null
+++ b/v2v/measure_disk.ml
@@ -0,0 +1,59 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open JSON_parser
+open Common_gettext.Gettext
+
+(* Run qemu-img measure on a disk. *)
+
+let measure ?format filename +  let cmd = ref [] in
+  List.push_back_list cmd [Guestfs_config.qemu_img; "measure"];
+  (match format with
+   | None -> ()
+   | Some format ->
+      List.push_back_list cmd ["-f"; format]
+  );
+  (* For use of -O qcow2 here, see this thread:
+   * https://www.redhat.com/archives/libguestfs/2018-August/thread.html#00142
+   *)
+  List.push_back_list cmd ["-O"; "qcow2"];
+  List.push_back cmd "--output=json";
+  List.push_back cmd filename;
+
+  let json = Filename.temp_file "v2vmeasure" ".json" in
+  unlink_on_exit json;
+
+  let fd = Unix.openfile json [O_WRONLY; O_CREAT; O_TRUNC] 0o600 in
+  if run_command ~stdout_fd:fd !cmd <> 0 then
+    error (f_"qemu-img measure failed, see earlier errors");
+  (* Note that run_command closes fd. *)
+
+  let json = read_whole_file json in
+  debug "qemu-img measure output: %s" json;
+  let json = json_parser_tree_parse json in
+  debug "qemu-img measure output parsed as: %s"
+        (JSON.string_of_doc ~fmt:JSON.Indented ["", json]);
+
+  (* We're expecting the tree to contain nodes:
+   * Dict [ "required", Int number; "fully-allocated", Int
number ]
+   * Of course the array could appear in any order.
+   *)
+  object_get_number "required" json
diff --git a/v2v/measure_disk.mli b/v2v/measure_disk.mli
new file mode 100644
index 000000000..efaa76105
--- /dev/null
+++ b/v2v/measure_disk.mli
@@ -0,0 +1,21 @@
+(* virt-v2v
+ * Copyright (C) 2018 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Run qemu-img measure on a disk. *)
+
+val measure : ?format:string -> string -> int64
diff --git a/v2v/test-v2v-print-estimate.sh b/v2v/test-v2v-print-estimate.sh
new file mode 100755
index 000000000..b04837ae1
--- /dev/null
+++ b/v2v/test-v2v-print-estimate.sh
@@ -0,0 +1,45 @@
+#!/bin/bash -
+# libguestfs virt-v2v test script
+# Copyright (C) 2018 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test --print-estimate option.
+
+set -e
+
+$TEST_FUNCTIONS
+skip_if_skipped
+skip_unless_phony_guest windows.img
+
+f=test-v2v-print-estimate.out
+rm -f $f
+
+echo "Actual:"
+du -s -B 1 ../test-data/phony-guests/windows.img
+
+$VG virt-v2v --debug-gc \
+    -i libvirtxml test-v2v-print-source.xml \
+    -o local -os $(pwd) \
+    --print-estimate \
+    --machine-readable=file:$f
+
+echo -n "Estimate: "
+cat $f
+
+# Check the output looks reasonable.
+grep -E 'total.*[[:digit:]]+' $f
+
+rm -f $f
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 1775200d3..884dfecaa 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -48,6 +48,8 @@ type mpstat = {
 
 let () = Random.self_init ()
 
+let sum = List.fold_left (+^) 0L
+
 let rec main ()    (* Handle the command line. *)
   let cmdline, input, output = parse_cmdline () in
@@ -156,6 +158,12 @@ let rec main ()    (match conversion_mode with
    | In_place -> ()
    | Copying overlays ->
+      (* Print copy size estimate and stop. *)
+      if cmdline.print_estimate then (
+        print_estimate overlays;
+        exit 0
+      );
+
       message (f_"Assigning disks to buses");
       let target_buses          Target_bus_assignment.target_bus_assignment
source guestcaps in
@@ -487,8 +495,6 @@ and do_fstrim g inspect   *     sdb final estimate size = 3
- (3*1.35/4) = 1.9875 GB
  *)
 and estimate_target_size mpstats overlays -  let sum = List.fold_left (+^) 0L
in
-
   (* (1) *)
   let fs_total_size      sum (
@@ -617,6 +623,25 @@ and get_target_formats cmdline output overlays       
format
   ) overlays
 
+and print_estimate overlays +  let estimates +    List.map (
+      fun { ov_overlay_file } ->
+        Measure_disk.measure ~format:"qcow2" ov_overlay_file
+    ) overlays in
+  let total = sum estimates in
+
+  match machine_readable () with
+  | None ->
+     List.iteri (fun i e -> printf "%d %Ld\n" (i+1) e) estimates;
+     printf "total %Ld\n" total
+  | Some {pr} ->
+     let json = [
+       "disks", JSON.List (List.map (fun i -> JSON.Int i)
estimates);
+       "total", JSON.Int total
+     ] in
+     pr "%s\n" (JSON.string_of_doc ~fmt:JSON.Indented json)
+
 (* Does the guest require UEFI on the target? *)
 and get_target_firmware inspect guestcaps source output    message
(f_"Checking if the guest needs BIOS or UEFI to boot");
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index fa7836659..9b48917a1 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -796,6 +796,37 @@ C<root>.
 You will get an error if virt-v2v is unable to mount/write to the
 Export Storage Domain.
 
+=item B<--print-estimate>
+
+Print the estimated size of the data which will be copied from the
+source disk(s) and stop.  One number (the size in bytes) is printed
+per disk, and a total:
+
+ $ virt-v2v --print-estimate
+ ...
+ 1 100000
+ 2 200000
+ total 300000
+
+With the I<--machine-readable> option you get JSON output which can be
+directed into a file or elsewhere:
+
+ $ virt-v2v --print-estimate --machine-readable=file:estimates
+ ...
+ $ cat estimates
+ {
+  "disks": [ 100000, 200000 ],
+  "total": 300000
+ }
+
+When using this option you must specify an output mode.  This is
+because virt-v2v has to perform the conversion in order to print the
+estimate, and the conversion depends on the output mode.  Using
+I<-o null> should be safe for most purposes.
+
+When this option is used along with I<--machine-readable> you can
+direct the output to an alternate file.
+
 =item B<--print-source>
 
 Print information about the source guest and stop.  This option is
-- 
2.18.0

On Thursday, 23 August 2018 20:13:21 CEST Richard W.M. Jones
wrote:
> This option prints the estimated size of the data that will be copied
> from the source disk.
> 
> Currently this overestimates by the size of the qcow2 header, but for
> real disk images that doesn't matter much.
> 
> For example:
> 
> $ virt-builder fedora-27
> $ virt-v2v -i disk fedora-27.img -o null --print-estimate
> [...]
> virt-v2v: This guest has virtio drivers installed.
> [  44.0] Mapping filesystem data to avoid copying unused and blank areas
> [  44.5] Closing the overlay
> 1 1047920640
> total 1047920640
Maybe having a slightly more user friendly output in non-machine
readable mode would be nice...
> +  let json = Filename.temp_file "v2vmeasure" ".json"
in
> +  unlink_on_exit json;
> +
> +  let fd = Unix.openfile json [O_WRONLY; O_CREAT; O_TRUNC] 0o600 in
  let chan, json = Filename.open_temp_file "v2vmeasure"
".json" in
  unlink_on_exit json;
  let fd = Unix.descr_of_out_channel chan in
  
> +  let json = read_whole_file json in
> +  debug "qemu-img measure output: %s" json;
> +  let json = json_parser_tree_parse json in
I just pushed json_parser_tree_parse_file, so that can be used instead
using read_whole_file.
> +echo -n "Estimate: "
> +cat $f
> +
> +# Check the output looks reasonable.
> +grep -E 'total.*[[:digit:]]+' $f
Maybe using a tool like jd to check the keys in the JSON dict would
be cleaner.

-- 
Pino Toscano