Libguestfs - Mar 2018 - [PATCH v2 0/5] Add --print-target with machine-readable version.

This adds --print-target.  In addition, v2 provides a machine-readable
version (in JSON format).

All of the record -> JSON boilerplate in this patch could be
eliminated if we moved the baseline to OCaml 4.02.

Rich.

The rebasing in commit 5dae9ca23dab90dfb890f6663aa4bacf1df31ced caused
this field to never get updated.
---
 v2v/v2v.ml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index f494defcf..75936c501 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -113,11 +113,12 @@ let rec main ()  
   let mpstats = get_mpstats g in
   check_guest_free_space mpstats;
-  (match conversion_mode with
-   | Copying (_, targets) ->
-       check_target_free_space mpstats source targets output
-   | In_place -> ()
-  );
+  let conversion_mode +    match conversion_mode with
+    | Copying (overlays, targets) ->
+       let targets = check_target_free_space mpstats source targets output in
+       Copying (overlays, targets)
+    | In_place -> In_place in
 
   (* Conversion. *)
   let guestcaps @@ -598,7 +599,8 @@ and check_target_free_space mpstats source
targets output    message (f_"Estimating space required on target for each
disk");
   let targets = estimate_target_size mpstats targets in
 
-  output#check_target_free_space source targets
+  output#check_target_free_space source targets;
+  targets
 
 (* Conversion. *)
 and do_convert g inspect source output rcaps -- 
2.13.2

This is analogous to --print-source, except that it prints the overlay
and target disk information.

The output looks like below.  Note there is one overlay and one target
section per disk.

Overlay and Target information (--print-target option):

             overlay file: /home/rjones/d/libguestfs/tmp/v2vovlc687fe.qcow2
      overlay device name: sda
overlay virtual disk size: 6442450944
  overlay source qemu URI: /var/tmp/fedora-27.img

          target file: [qemu] json:{ "file.driver": "nbd",
"file.path":
"/home/rjones/d/libguestfs/tmp/rhvupload.IWrzO6/nbdkit0.sock",
"file.export": "/" }
        target format: raw
target estimated size: 2274060540
---
 v2v/cmdline.ml   | 14 +++++++++++++-
 v2v/cmdline.mli  |  1 +
 v2v/types.ml     | 20 +++++++-------------
 v2v/v2v.ml       | 17 +++++++++++++++++
 v2v/virt-v2v.pod |  5 +++++
 5 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 4f651825a..6aba4afb5 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -44,6 +44,7 @@ type cmdline = {
   output_format : string option;
   output_name : string option;
   print_source : bool;
+  print_target : bool;
   root_choice : root_choice;
 }
 
@@ -53,6 +54,7 @@ let parse_cmdline ()    let do_copy = ref true in
   let machine_readable = ref false in
   let print_source = ref false in
+  let print_target = ref false in
   let qemu_boot = ref false in
 
   let input_conn = ref None in
@@ -233,6 +235,8 @@ let parse_cmdline ()                                     
s_"Use password from file";
     [ L"print-source" ], Getopt.Set print_source,
                                     s_"Print source and stop";
+    [ L"print-target" ], Getopt.Set print_target,
+                                    s_"Print target and stop";
     [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o
qemu only)";
     [ L"rhv-cafile" ], Getopt.String ("ca.pem",
set_string_option_once "--rhv-cafile" rhv_cafile),
                                     s_"For -o rhv-upload, set ‘ca.pem’
file";
@@ -330,6 +334,7 @@ read the man page virt-v2v(1).
   let output_storage = !output_storage in
   let password_file = !password_file in
   let print_source = !print_source in
+  let print_target = !print_target in
   let qemu_boot = !qemu_boot in
   let rhv_cafile = !rhv_cafile in
   let rhv_direct = !rhv_direct in
@@ -371,6 +376,12 @@ read the man page virt-v2v(1).
     exit 0
   );
 
+  (* Some options cannot be used with --in-place. *)
+  if in_place then (
+    if print_target then
+      error (f_"--in-place and --print-target cannot be used
together")
+  );
+
   (* Parse out the password from the password file. *)
   let password      match password_file with
@@ -620,6 +631,7 @@ read the man page virt-v2v(1).
     do_copy = do_copy; in_place = in_place; network_map = network_map;
     output_alloc = output_alloc; output_format = output_format;
     output_name = output_name;
-    print_source = print_source; root_choice = root_choice;
+    print_source = print_source; print_target;
+    root_choice = root_choice;
   },
   input, output
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 7d88f0f54..0e1d54f40 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -42,6 +42,7 @@ type cmdline = {
   output_format : string option;
   output_name : string option;
   print_source : bool;
+  print_target : bool;
   root_choice : Types.root_choice;
 }
 
diff --git a/v2v/types.ml b/v2v/types.ml
index b89bd2fe2..c1f36fdb1 100644
--- a/v2v/types.ml
+++ b/v2v/types.ml
@@ -281,11 +281,10 @@ type overlay = {
 }
 
 let string_of_overlay ov -  sprintf "\
-ov_overlay_file = %s
-ov_sd = %s
-ov_virtual_size = %Ld
-ov_source = %s
+  sprintf "             overlay file: %s
+      overlay device name: %s
+overlay virtual disk size: %Ld
+  overlay source qemu URI: %s
 "
     ov.ov_overlay_file
     ov.ov_sd
@@ -304,12 +303,9 @@ and target_file    | TargetURI of string
 
 let string_of_target t -  sprintf "\
-target_file = %s
-target_format = %s
-target_estimated_size = %s
-target_overlay = %s
-target_overlay.ov_source = %s
+  sprintf "          target file: %s
+        target format: %s
+target estimated size: %s
 "
     (match t.target_file with
      | TargetFile s -> "[file] " ^ s
@@ -317,8 +313,6 @@ target_overlay.ov_source = %s
     t.target_format
     (match t.target_estimated_size with
     | None -> "None" | Some i -> Int64.to_string i)
-    t.target_overlay.ov_overlay_file
-    t.target_overlay.ov_source.s_qemu_uri
 
 type target_firmware = TargetBIOS | TargetUEFI
 
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index 75936c501..abb531c6f 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -150,6 +150,22 @@ let rec main ()    g#shutdown ();
   g#close ();
 
+  (match conversion_mode with
+   | In_place -> ()
+   | Copying (overlays, targets) ->
+      (* Print overlays/targets and stop. *)
+      if cmdline.print_target then (
+        printf (f_"Overlay and Target information (--print-target
option):\n");
+        printf "\n";
+        List.iter (
+          fun (ov, t) ->
+            printf "%s\n" (string_of_overlay ov);
+            printf "%s\n" (string_of_target t)
+        ) (List.combine overlays targets);
+        exit 0
+      )
+  );
+
   (* Copy overlays to target (for [--in-place] this does nothing). *)
   (match conversion_mode with
    | In_place -> ()
@@ -685,6 +701,7 @@ and copy_targets cmdline targets input output           
message (f_"Copying disk %d/%d to qemu URI %s (%s)")
                   (i+1) nr_disks s t.target_format
       );
+      debug "%s" (string_of_overlay t.target_overlay);
       debug "%s" (string_of_target t);
 
       (* We noticed that qemu sometimes corrupts the qcow2 file on
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index faf8c0ba2..bbd6985e4 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -631,6 +631,11 @@ Print information about the source guest and stop.  This
option is
 useful when you are setting up network and bridge maps.
 See L</NETWORKS AND BRIDGES>.
 
+=item B<--print-target>
+
+Print information about the target disk(s) and overlay file(s), and
+stop.
+
 =item B<--qemu-boot>
 
 When using I<-o qemu> only, this boots the guest immediately after
-- 
2.13.2

See:
https://forge.ocamlcore.org/docman/view.php/77/112/leroy-cug2010.pdf
---
 v2v/cmdline.ml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 6aba4afb5..fdd0f2614 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -627,11 +627,9 @@ read the man page virt-v2v(1).
       output_format, output_alloc in
 
   {
-    compressed = compressed; debug_overlays = debug_overlays;
-    do_copy = do_copy; in_place = in_place; network_map = network_map;
-    output_alloc = output_alloc; output_format = output_format;
-    output_name = output_name;
-    print_source = print_source; print_target;
-    root_choice = root_choice;
+    compressed; debug_overlays; do_copy; in_place; network_map;
+    output_alloc; output_format; output_name;
+    print_source; print_target;
+    root_choice;
   },
   input, output
-- 
2.13.2

Produces machine-readable output delimited by special markers
__MACHINEBEGIN__
__MACHINEEND__
---
 common/mltools/tools_utils.ml  |  8 ++++++++
 common/mltools/tools_utils.mli | 11 +++++++++++
 2 files changed, 19 insertions(+)

diff --git a/common/mltools/tools_utils.ml b/common/mltools/tools_utils.ml
index 09f1bb544..f7abd5c3e 100644
--- a/common/mltools/tools_utils.ml
+++ b/common/mltools/tools_utils.ml
@@ -102,6 +102,14 @@ let debug fs    let display str = if verbose () then
prerr_endline str in
   ksprintf display fs
 
+let machine_output fs +  let display str +    print_endline
"__MACHINEBEGIN__";
+    print_endline str;
+    print_endline "__MACHINEEND__"
+  in
+  ksprintf display fs
+
 (* Common function to create a new Guestfs handle, with common options
  * (e.g. debug, tracing) already set.
  *)
diff --git a/common/mltools/tools_utils.mli b/common/mltools/tools_utils.mli
index dac6b4120..d40e0aa9b 100644
--- a/common/mltools/tools_utils.mli
+++ b/common/mltools/tools_utils.mli
@@ -36,6 +36,17 @@ val debug : ('a, unit, string, unit) format4 -> 'a
     the command line.  As with libguestfs debugging messages, it is
     sent to [stderr]. *)
 
+val machine_output : ('a, unit, string, unit) format4 -> 'a
+(** Output meant to be parsed by another program.
+
+    It is printed with no wrapping or other interpretation as:
+    {v
+    __MACHINEBEGIN__
+    <<output, possibly multiple lines>>
+    __MACHINEEND__
+    v}
+ *)
+
 val open_guestfs : ?identifier:string -> unit -> Guestfs.guestfs
 (** Common function to create a new Guestfs handle, with common options
     (e.g. debug, tracing) already set.
-- 
2.13.2

The output will look similar to this:

__MACHINEBEGIN__
{
  "targets": [
    {
      "qemu_uri": "json:{ \"file.driver\":
\"nbd\", \"file.path\":
\"/home/rjones/d/libguestfs/tmp/rhvupload.VVGewv/nbdkit0.sock\",
\"file.export\": \"/\" }",
      "format": "raw",
      "estimated_size": 2274060540
    }
  ],
  "overlays": [
    {
      "file":
"/home/rjones/d/libguestfs/tmp/v2vovl827733.qcow2",
      "sd": "sda",
      "virtual_size": 6442450944
    }
  ]
}
__MACHINEEND__
---
 v2v/cmdline.ml   |  4 +++-
 v2v/cmdline.mli  |  1 +
 v2v/types.ml     | 18 ++++++++++++++++++
 v2v/types.mli    |  2 ++
 v2v/v2v.ml       | 30 +++++++++++++++++++++++-------
 v2v/virt-v2v.pod |  6 ++++++
 6 files changed, 53 insertions(+), 8 deletions(-)

diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index fdd0f2614..25d26b946 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -39,6 +39,7 @@ type cmdline = {
   debug_overlays : bool;
   do_copy : bool;
   in_place : bool;
+  machine_readable : bool;
   network_map : string NetworkMap.t;
   output_alloc : output_allocation;
   output_format : string option;
@@ -627,7 +628,8 @@ read the man page virt-v2v(1).
       output_format, output_alloc in
 
   {
-    compressed; debug_overlays; do_copy; in_place; network_map;
+    compressed; debug_overlays; do_copy; in_place;
+    machine_readable; network_map;
     output_alloc; output_format; output_name;
     print_source; print_target;
     root_choice;
diff --git a/v2v/cmdline.mli b/v2v/cmdline.mli
index 0e1d54f40..765f63a2d 100644
--- a/v2v/cmdline.mli
+++ b/v2v/cmdline.mli
@@ -37,6 +37,7 @@ type cmdline = {
   debug_overlays : bool;
   do_copy : bool;
   in_place : bool;
+  machine_readable : bool;
   network_map : string NetworkMap.t;
   output_alloc : Types.output_allocation;
   output_format : string option;
diff --git a/v2v/types.ml b/v2v/types.ml
index c1f36fdb1..eeec077ee 100644
--- a/v2v/types.ml
+++ b/v2v/types.ml
@@ -291,6 +291,11 @@ overlay virtual disk size: %Ld
     ov.ov_virtual_size
     ov.ov_source.s_qemu_uri
 
+let json_of_overlay ov +  [ "file", JSON.String ov.ov_overlay_file;
+    "sd", JSON.String ov.ov_sd;
+    "virtual_size", JSON.Int64 ov.ov_virtual_size ]
+
 type target = {
   target_file : target_file;
   target_format : string;
@@ -314,6 +319,19 @@ target estimated size: %s
     (match t.target_estimated_size with
     | None -> "None" | Some i -> Int64.to_string i)
 
+let json_of_target t +  let json = ref [] in
+  (match t.target_file with
+   | TargetFile s ->
+      List.push_back json ("file", JSON.String s)
+   | TargetURI s ->
+      List.push_back json ("qemu_uri", JSON.String s)
+  );
+  List.push_back json ("format", JSON.String t.target_format);
+  Option.may (fun i -> List.push_back json ("estimated_size",
JSON.Int64 i))
+             t.target_estimated_size;
+  !json
+
 type target_firmware = TargetBIOS | TargetUEFI
 
 let string_of_target_firmware = function
diff --git a/v2v/types.mli b/v2v/types.mli
index a432e167c..d3dfebb7f 100644
--- a/v2v/types.mli
+++ b/v2v/types.mli
@@ -185,6 +185,7 @@ type overlay = {
 (** Overlay disk. *)
 
 val string_of_overlay : overlay -> string
+val json_of_overlay : overlay -> JSON.doc
 
 (** {2 Target disks} *)
 
@@ -208,6 +209,7 @@ and target_file    | TargetURI of string      (** Target is
a QEMU URI. *)
 
 val string_of_target : target -> string
+val json_of_target : target -> JSON.doc
 
 (** {2 Guest firmware} *)
 
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
index abb531c6f..63879a469 100644
--- a/v2v/v2v.ml
+++ b/v2v/v2v.ml
@@ -155,13 +155,28 @@ let rec main ()     | Copying (overlays, targets) ->
       (* Print overlays/targets and stop. *)
       if cmdline.print_target then (
-        printf (f_"Overlay and Target information (--print-target
option):\n");
-        printf "\n";
-        List.iter (
-          fun (ov, t) ->
-            printf "%s\n" (string_of_overlay ov);
-            printf "%s\n" (string_of_target t)
-        ) (List.combine overlays targets);
+        if not cmdline.machine_readable then (
+          printf (f_"Overlay and Target information (--print-target
option):\n");
+          printf "\n";
+          List.iter (
+            fun (ov, t) ->
+              printf "%s\n" (string_of_overlay ov);
+              printf "%s\n" (string_of_target t)
+          ) (List.combine overlays targets);
+        )
+        else (
+          let json = [
+            "targets",
+              JSON.List (
+                List.map (fun t -> JSON.Dict (json_of_target t)) targets
+              );
+            "overlays",
+              JSON.List (
+                List.map (fun ov -> JSON.Dict (json_of_overlay ov)) overlays
+              );
+          ] in
+          machine_output "%s" (JSON.string_of_doc ~fmt:JSON.Indented
json)
+        );
         exit 0
       )
   );
@@ -200,6 +215,7 @@ and open_source cmdline input  
   (* Print source and stop. *)
   if cmdline.print_source then (
+    (* XXX Implement machine-readable version. *)
     printf (f_"Source guest information (--print-source option):\n");
     printf "\n";
     printf "%s\n" (string_of_source source);
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index bbd6985e4..5c7e563bf 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -636,6 +636,12 @@ See L</NETWORKS AND BRIDGES>.
 Print information about the target disk(s) and overlay file(s), and
 stop.
 
+To obtain machine-readable JSON output use:
+
+ virt-v2v ... --print-target --machine-readable
+
+and parse the lines between C<__MACHINEBEGIN__> and
C<__MACHINEEND__>.
+
 =item B<--qemu-boot>
 
 When using I<-o qemu> only, this boots the guest immediately after
-- 
2.13.2

On Friday, 16 March 2018 14:30:41 CET Richard W.M. Jones
wrote:
> See:
> https://forge.ocamlcore.org/docman/view.php/77/112/leroy-cug2010.pdf
Followup of commit c7651744da455a00a7abeb930621c50bfb23c40c.

-- 
Pino Toscano

On Friday, 16 March 2018 14:30:42 CET Richard W.M. Jones
wrote:
> Produces machine-readable output delimited by special markers
> __MACHINEBEGIN__
> __MACHINEEND__
> ---
While I like the idea of machine readable output (with a standard
format even), I'm not sure about this way of printing it together
with non-machine readable stuff such as debug and trace.

The problem is that the debug output goes to stdout, and the trace
to stderr, so v2v users would need to "drain" either of them to get
the machine readable markers, then drain until the end marker, and so
on to the next marker (until the end of the execution).

IMHO a better idea is to send the machine readable stuff to something
else, e.g. using
  --machine-readable-output FILE
  --machine-readable-FD FD
This way, v2v users will just get the right data with no need to filter
anything else.  The same mechanism can be extended to --print-source,
and to the status/progress output (avoiding the need to parse even
shell colors).

-- 
Pino Toscano