Libguestfs - Jan 2014 - [PATCH 0/3] Add JSON output for virt-builder

Hi,

This small patch serie adds a JSON output for virt-builder.
This way it is possible to parse the list of available templates,
with no need to parse the unstructured and possibly changing short and
long outputs of virt-builder.


Pino Toscano (3):
  builder: small refactor of the list output
  builder: add --list-format
  builder: add a JSON output for --list

 builder/builder.ml                |   4 +-
 builder/cmdline.ml                |  19 +++--
 builder/list_entries.ml           | 142 ++++++++++++++++++++++++++++----------
 builder/list_entries.mli          |   2 +-
 builder/test-virt-builder-list.sh |  67 ++++++++++++++++++
 builder/virt-builder.pod          |  34 ++++++++-
 6 files changed, 222 insertions(+), 46 deletions(-)

-- 
1.8.3.1

Switch from a boolean for the short/long list output to labels for the
actual format. Also, split the output of each list format to an own
function for easier maintaineance.
---
 builder/builder.ml       |  4 +--
 builder/cmdline.ml       |  9 +++---
 builder/list_entries.ml  | 82 +++++++++++++++++++++++++++---------------------
 builder/list_entries.mli |  2 +-
 4 files changed, 54 insertions(+), 43 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 861e029..bb0b108 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -37,7 +37,7 @@ let main ()    (* Command line argument parsing - see
cmdline.ml. *)
   let mode, arg,
     attach, cache, check_signature, curl, debug, delete, delete_on_failure,
-    edit, firstboot, run, format, gpg, hostname, install, list_long, links,
+    edit, firstboot, run, format, gpg, hostname, install, list_format, links,
     memsize, mkdirs,
     network, output, password_crypto, quiet, root_password, scrub,
     scrub_logfile, size, smp, sources, sync, timezone, update, upload,
@@ -148,7 +148,7 @@ let main ()    let mode      match mode with
     | `List ->                          (* --list *)
-      List_entries.list_entries ~list_long ~sources index;
+      List_entries.list_entries ~list_format ~sources index;
       exit 0
 
     | `Print_cache ->                   (* --print-cache *)
diff --git a/builder/cmdline.ml b/builder/cmdline.ml
index 67b142a..f199f03 100644
--- a/builder/cmdline.ml
+++ b/builder/cmdline.ml
@@ -130,7 +130,8 @@ let parse_cmdline ()      links := (target, lns) :: !links
   in
 
-  let list_long = ref false in
+  let list_format = ref `Short in
+  let list_set_long () = list_format := `Long in
 
   let memsize = ref None in
   let set_memsize arg = memsize := Some arg in
@@ -255,7 +256,7 @@ let parse_cmdline ()      "--link",    Arg.String
add_link,       "target:link.." ^ " " ^ s_"Create
symbolic links";
     "-l",        Arg.Unit list_mode,        " " ^
s_"List available templates";
     "--list",    Arg.Unit list_mode,        ditto;
-    "--long",    Arg.Set list_long,         ditto;
+    "--long",    Arg.Unit list_set_long,    " " ^
s_"List available templates, in long textual form";
     "--no-logfile", Arg.Set scrub_logfile,  " " ^
s_"Scrub build log file";
     "--long-options", Arg.Unit display_long_options, " " ^
s_"List long options";
     "-m",        Arg.Int set_memsize,       "mb" ^ "
" ^ s_"Set memory size";
@@ -329,7 +330,7 @@ read the man page virt-builder(1).
   let gpg = !gpg in
   let hostname = !hostname in
   let install = List.rev !install in
-  let list_long = !list_long in
+  let list_format = !list_format in
   let links = List.rev !links in
   let memsize = !memsize in
   let mkdirs = List.rev !mkdirs in
@@ -443,7 +444,7 @@ read the man page virt-builder(1).
 
   mode, arg,
   attach, cache, check_signature, curl, debug, delete, delete_on_failure,
-  edit, firstboot, run, format, gpg, hostname, install, list_long, links,
+  edit, firstboot, run, format, gpg, hostname, install, list_format, links,
   memsize, mkdirs,
   network, output, password_crypto, quiet, root_password, scrub,
   scrub_logfile, size, smp, sources, sync, timezone, update, upload,
diff --git a/builder/list_entries.ml b/builder/list_entries.ml
index 87001c0..97ab201 100644
--- a/builder/list_entries.ml
+++ b/builder/list_entries.ml
@@ -21,15 +21,35 @@ open Common_utils
 
 open Printf
 
-let list_entries ?(list_long = false) ~sources index -  if list_long then (
-    List.iter (
-      fun (source, fingerprint) ->
-        printf (f_"Source URI: %s\n") source;
-        printf (f_"Fingerprint: %s\n") fingerprint;
+let rec list_entries ~list_format ~sources index +  match list_format with
+  | `Short -> list_entries_short index
+  | `Long -> list_entries_long ~sources index
+
+and list_entries_short index +  List.iter (
+    fun (name, { Index_parser.printable_name = printable_name;
+                 size = size;
+                 compressed_size = compressed_size;
+                 notes = notes;
+                 hidden = hidden }) ->
+      if not hidden then (
+        printf "%-24s" name;
+        (match printable_name with
+        | None -> ()
+        | Some s -> printf " %s" s
+        );
         printf "\n"
-    ) sources
-  );
+      )
+  ) index
+
+and list_entries_long ~sources index +  List.iter (
+    fun (source, fingerprint) ->
+      printf (f_"Source URI: %s\n") source;
+      printf (f_"Fingerprint: %s\n") fingerprint;
+      printf "\n"
+  ) sources;
 
   List.iter (
     fun (name, { Index_parser.printable_name = printable_name;
@@ -38,33 +58,23 @@ let list_entries ?(list_long = false) ~sources index        
notes = notes;
                  hidden = hidden }) ->
       if not hidden then (
-        if not list_long then (         (* Short *)
-          printf "%-24s" name;
-          (match printable_name with
-          | None -> ()
-          | Some s -> printf " %s" s
-          );
-          printf "\n"
-        )
-        else (                          (* Long *)
-          printf "%-24s %s\n" "os-version:" name;
-          (match printable_name with
-          | None -> ()
-          | Some name -> printf "%-24s %s\n" (s_"Full
name:") name;
-          );
-          printf "%-24s %s\n" (s_"Minimum/default size:")
(human_size size);
-          (match compressed_size with
-          | None -> ()
-          | Some size ->
-            printf "%-24s %s\n" (s_"Download size:")
(human_size size);
-          );
-          (match notes with
-          | None -> ()
-          | Some notes ->
-            printf "\n";
-            printf (f_"Notes:\n\n%s\n") notes
-          );
-          printf "\n"
-        )
+        printf "%-24s %s\n" "os-version:" name;
+        (match printable_name with
+        | None -> ()
+        | Some name -> printf "%-24s %s\n" (s_"Full
name:") name;
+        );
+        printf "%-24s %s\n" (s_"Minimum/default size:")
(human_size size);
+        (match compressed_size with
+        | None -> ()
+        | Some size ->
+          printf "%-24s %s\n" (s_"Download size:")
(human_size size);
+        );
+        (match notes with
+        | None -> ()
+        | Some notes ->
+          printf "\n";
+          printf (f_"Notes:\n\n%s\n") notes
+        );
+        printf "\n"
       )
   ) index
diff --git a/builder/list_entries.mli b/builder/list_entries.mli
index d9486b0..41d0bff 100644
--- a/builder/list_entries.mli
+++ b/builder/list_entries.mli
@@ -16,4 +16,4 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val list_entries : ?list_long:bool -> sources:(string * string) list ->
Index_parser.index -> unit
+val list_entries : list_format:([ `Short | `Long ]) -> sources:(string *
string) list -> Index_parser.index -> unit
-- 
1.8.3.1

Add a --list-format which allows to choose which in format should be the
output of --list.
---
 builder/cmdline.ml       | 11 ++++++++++-
 builder/virt-builder.pod | 24 +++++++++++++++++++++---
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/builder/cmdline.ml b/builder/cmdline.ml
index f199f03..6d6439f 100644
--- a/builder/cmdline.ml
+++ b/builder/cmdline.ml
@@ -132,6 +132,13 @@ let parse_cmdline ()  
   let list_format = ref `Short in
   let list_set_long () = list_format := `Long in
+  let list_set_format arg +    list_format := match arg with
+    | "short" -> `Short
+    | "long" -> `Long
+    | fmt ->
+      eprintf (f_"%s: invalid --list-format type '%s', see the man
page.\n") prog fmt;
+      exit 1 in
 
   let memsize = ref None in
   let set_memsize arg = memsize := Some arg in
@@ -256,7 +263,9 @@ let parse_cmdline ()      "--link",    Arg.String
add_link,       "target:link.." ^ " " ^ s_"Create
symbolic links";
     "-l",        Arg.Unit list_mode,        " " ^
s_"List available templates";
     "--list",    Arg.Unit list_mode,        ditto;
-    "--long",    Arg.Unit list_set_long,    " " ^
s_"List available templates, in long textual form";
+    "--long",    Arg.Unit list_set_long,    " " ^
s_"Shortcut for --list-format short";
+    "--list-format", Arg.String list_set_format,
+                                            "short|long" ^ "
" ^ s_"Set the format for --list (default: short)";
     "--no-logfile", Arg.Set scrub_logfile,  " " ^
s_"Scrub build log file";
     "--long-options", Arg.Unit display_long_options, " " ^
s_"List long options";
     "-m",        Arg.Int set_memsize,       "mb" ^ "
" ^ s_"Set memory size";
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 9cbfbab..05abcc6 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -31,7 +31,7 @@ virt-builder - Build virtual machine images quickly
     [--firstboot SCRIPT] [--firstboot-command 'CMD ARGS ...']
     [--firstboot-install PKG,[PKG...]]
 
- virt-builder -l|--list [--long]
+ virt-builder -l|--list [--long] [--list-format short|long]
 
  virt-builder --notes os-version
 
@@ -374,12 +374,30 @@ pointing at C<TARGET>.
 
 =item B<--list>
 
+=item B<--list --format> format
+
 =item B<--list --long>
 
 List available templates.
 
-The alternative I<--list --long> form shows lots more details about
-each operating system option.
+It is possible to choose with I<--format> the output format for the list
+templates:
+
+=over 4
+
+=item B<short>
+
+The default format, prints only the template identifier and, next to it,
+its short description.
+
+=item B<long>
+
+Prints a textual list with the details of the available sources, followed
+by the details of the available templates.
+
+=back
+
+I<--long> is a shorthand for the C<long> format.
 
 See also: I<--source>, I<--notes>, L</CREATING YOUR OWN
TEMPLATES>.
 
-- 
1.8.3.1

Simple JSON output for sources and templates, to be able to query them
with no need to parse unstructured outputs like the "--list-format
long"
one.
---
 builder/cmdline.ml                |  3 +-
 builder/list_entries.ml           | 60 +++++++++++++++++++++++++++++++++++
 builder/list_entries.mli          |  2 +-
 builder/test-virt-builder-list.sh | 67 +++++++++++++++++++++++++++++++++++++++
 builder/virt-builder.pod          | 12 ++++++-
 5 files changed, 141 insertions(+), 3 deletions(-)

diff --git a/builder/cmdline.ml b/builder/cmdline.ml
index 6d6439f..e3b1484 100644
--- a/builder/cmdline.ml
+++ b/builder/cmdline.ml
@@ -136,6 +136,7 @@ let parse_cmdline ()      list_format := match arg with
     | "short" -> `Short
     | "long" -> `Long
+    | "json" -> `Json
     | fmt ->
       eprintf (f_"%s: invalid --list-format type '%s', see the man
page.\n") prog fmt;
       exit 1 in
@@ -265,7 +266,7 @@ let parse_cmdline ()      "--list",    Arg.Unit
list_mode,        ditto;
     "--long",    Arg.Unit list_set_long,    " " ^
s_"Shortcut for --list-format short";
     "--list-format", Arg.String list_set_format,
-                                            "short|long" ^ "
" ^ s_"Set the format for --list (default: short)";
+                                            "short|long|json" ^
" " ^ s_"Set the format for --list (default: short)";
     "--no-logfile", Arg.Set scrub_logfile,  " " ^
s_"Scrub build log file";
     "--long-options", Arg.Unit display_long_options, " " ^
s_"List long options";
     "-m",        Arg.Int set_memsize,       "mb" ^ "
" ^ s_"Set memory size";
diff --git a/builder/list_entries.ml b/builder/list_entries.ml
index 97ab201..7369e6c 100644
--- a/builder/list_entries.ml
+++ b/builder/list_entries.ml
@@ -25,6 +25,7 @@ let rec list_entries ~list_format ~sources index    match
list_format with
   | `Short -> list_entries_short index
   | `Long -> list_entries_long ~sources index
+  | `Json -> list_entries_json ~sources index
 
 and list_entries_short index    List.iter (
@@ -78,3 +79,62 @@ and list_entries_long ~sources index          printf
"\n"
       )
   ) index
+
+and list_entries_json ~sources index +  let trailing_comma index size +    if
index = size - 1 then "" else "," in
+  let json_string_of_bool b +    if b then "true" else
"false" in
+  let json_string_escape str +    let res = ref "" in
+    for i = 0 to String.length str - 1 do
+      res := !res ^ (match str.[i] with
+        | '"' -> "\\\""
+        | '\\' -> "\\\\"
+        | '\b' -> "\\b"
+        | '\n' -> "\\n"
+        | '\r' -> "\\r"
+        | '\t' -> "\\t"
+        | c -> String.make 1 c)
+    done;
+    !res in
+  let json_optional_printf_string key value +    match value with
+    | None -> ()
+    | Some str ->
+      printf "    \"%s\": \"%s\",\n" key
(json_string_escape str) in
+  let json_optional_printf_int64 key value +    match value with
+    | None -> ()
+    | Some n ->
+      printf "    \"%s\": \"%Ld\",\n" key n in
+
+  printf "{\n";
+  printf "  \"version\": %d,\n" 1;
+  printf "  \"sources\": [\n";
+  iteri (
+    fun i (source, fingerprint) ->
+      printf "  {\n";
+      printf "    \"uri\": \"%s\",\n" source;
+      printf "    \"fingerprint\": \"%s\"\n"
fingerprint;
+      printf "  }%s\n" (trailing_comma i (List.length sources))
+  ) sources;
+  printf "  ],\n";
+  printf "  \"templates\": [\n";
+  iteri (
+    fun i (name, { Index_parser.printable_name = printable_name;
+                   size = size;
+                   compressed_size = compressed_size;
+                   notes = notes;
+                   hidden = hidden }) ->
+      printf "  {\n";
+      printf "    \"os-version\": \"%s\",\n"
name;
+      json_optional_printf_string "full-name" printable_name;
+      printf "    \"size\": %Ld,\n" size;
+      json_optional_printf_int64 "compressed-size" compressed_size;
+      json_optional_printf_string "notes" notes;
+      printf "    \"hidden\": %s\n" (json_string_of_bool
hidden);
+      printf "  }%s\n" (trailing_comma i (List.length index))
+  ) index;
+  printf "  ]\n";
+ printf "}\n"
diff --git a/builder/list_entries.mli b/builder/list_entries.mli
index 41d0bff..e7c32f1 100644
--- a/builder/list_entries.mli
+++ b/builder/list_entries.mli
@@ -16,4 +16,4 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val list_entries : list_format:([ `Short | `Long ]) -> sources:(string *
string) list -> Index_parser.index -> unit
+val list_entries : list_format:([ `Short | `Long | `Json ]) ->
sources:(string * string) list -> Index_parser.index -> unit
diff --git a/builder/test-virt-builder-list.sh
b/builder/test-virt-builder-list.sh
index 083c035..c3b791f 100755
--- a/builder/test-virt-builder-list.sh
+++ b/builder/test-virt-builder-list.sh
@@ -103,3 +103,70 @@ Phony Windows look-alike used for testing." ]; then
     echo "$long_list"
     exit 1
 fi
+
+json_list=$(./virt-builder --no-check-signature --no-cache --list --list-format
json)
+
+if [ "$json_list" != "{
+  \"version\": 1,
+  \"sources\": [
+  {
+    \"uri\": \"$VIRT_BUILDER_SOURCE\",
+    \"fingerprint\": \"F777 4FB1 AD07 4A7E 8C87 67EA 9173 8F73
E1B7 68A0\"
+  }
+  ],
+  \"templates\": [
+  {
+    \"os-version\": \"phony-debian\",
+    \"full-name\": \"Phony Debian\",
+    \"size\": 536870912,
+    \"notes\": \"Phony Debian look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora\",
+    \"full-name\": \"Phony Fedora\",
+    \"size\": 1073741824,
+    \"notes\": \"Phony Fedora look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-qcow2\",
+    \"full-name\": \"Phony Fedora qcow2\",
+    \"size\": 1073741824,
+    \"notes\": \"Phony Fedora look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-qcow2-uncompressed\",
+    \"full-name\": \"Phony Fedora qcow2 uncompressed\",
+    \"size\": 1073741824,
+    \"notes\": \"Phony Fedora look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-no-format\",
+    \"full-name\": \"Phony Fedora\",
+    \"size\": 1073741824,
+    \"notes\": \"Phony Fedora look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-ubuntu\",
+    \"full-name\": \"Phony Ubuntu\",
+    \"size\": 536870912,
+    \"notes\": \"Phony Ubuntu look-alike used for
testing.\",
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-windows\",
+    \"full-name\": \"Phony Windows\",
+    \"size\": 536870912,
+    \"notes\": \"Phony Windows look-alike used for
testing.\",
+    \"hidden\": false
+  }
+  ]
+}" ]; then
+    echo "$0: unexpected --list --format json output:"
+    echo "$json_list"
+    exit 1
+fi
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 05abcc6..ded045b 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -31,7 +31,7 @@ virt-builder - Build virtual machine images quickly
     [--firstboot SCRIPT] [--firstboot-command 'CMD ARGS ...']
     [--firstboot-install PKG,[PKG...]]
 
- virt-builder -l|--list [--long] [--list-format short|long]
+ virt-builder -l|--list [--long] [--list-format short|long|json]
 
  virt-builder --notes os-version
 
@@ -395,6 +395,16 @@ its short description.
 Prints a textual list with the details of the available sources, followed
 by the details of the available templates.
 
+=item B<json>
+
+Prints a JSON object with the details of the available sources and
+the details of the available templates.
+
+The C<version> key in the main object represents the "compatibility
version",
+and it is bumped every time the resulting JSON output is incompatible with
+the previous versions (for example the structure has changed, or non-optional
+keys are no more present).
+
 =back
 
 I<--long> is a shorthand for the C<long> format.
-- 
1.8.3.1

On Thu, Jan 16, 2014 at 03:04:31PM +0100, Pino Toscano
wrote:
> Add a --list-format which allows to choose which in format should be the
> output of --list.
> ---
>  builder/cmdline.ml       | 11 ++++++++++-
>  builder/virt-builder.pod | 24 +++++++++++++++++++++---
>  2 files changed, 31 insertions(+), 4 deletions(-)
> 
> diff --git a/builder/cmdline.ml b/builder/cmdline.ml
> index f199f03..6d6439f 100644
> --- a/builder/cmdline.ml
> +++ b/builder/cmdline.ml
> @@ -132,6 +132,13 @@ let parse_cmdline () >  
>    let list_format = ref `Short in
>    let list_set_long () = list_format := `Long in
> +  let list_set_format arg > +    list_format := match arg with
> +    | "short" -> `Short
> +    | "long" -> `Long
> +    | fmt ->
> +      eprintf (f_"%s: invalid --list-format type '%s', see
the man page.\n") prog fmt;
> +      exit 1 in
It's possible to write this as:

  let list_set_format = function
  | "short" -> list_format := `Short
  etc.

Not sure if it will make any difference here.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines.  Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://people.redhat.com/~rjones/virt-top

On Thu, Jan 16, 2014 at 03:04:32PM +0100, Pino Toscano
wrote:
> +  let json_optional_printf_string key value > +    match value with
> +    | None -> ()
> +    | Some str ->
> +      printf "    \"%s\": \"%s\",\n" key
(json_string_escape str) in
> +  let json_optional_printf_int64 key value > +    match value with
> +    | None -> ()
> +    | Some n ->
> +      printf "    \"%s\": \"%Ld\",\n" key n
in
Again you can shorten these slightly by:

  let json_optional_printf_string key = function
  | None -> ()
  | Some str -> ...

Anyway, *ACK* to this patch series.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v