Libguestfs - Feb 2019 - [PATCH 2/3] v2v: add Var_expander

This series adds a new output mode for virt-v2v, called -o json.
It produces local files, just like -o local, although the metadata
produced is a JSON file with data that v2v collected in the conversion
process.  This can be useful for converting to unsupported destinations,
still based on QEMU/KVM.

In addition to a simple different metadata, it offers a way to relocate
the disks, with %{...}-like variables (only 3 added ATM, more can be
added) to change their paths depending on data of the guest/disks.

Thanks,


Pino Toscano (3):
  common/mlpcre: add offset flag for PCRE.matches
  v2v: add Var_expander
  v2v: add -o json output mode

 .gitignore                    |   1 +
 common/mlpcre/PCRE.ml         |   2 +-
 common/mlpcre/PCRE.mli        |   5 +-
 common/mlpcre/pcre-c.c        |  16 +-
 common/mlpcre/pcre_tests.ml   |  15 +-
 v2v/Makefile.am               |  36 +++-
 v2v/cmdline.ml                |  29 +++
 v2v/create_json.ml            | 348 ++++++++++++++++++++++++++++++++++
 v2v/create_json.mli           |  29 +++
 v2v/dummy.c                   |   2 +
 v2v/output_json.ml            | 116 ++++++++++++
 v2v/output_json.mli           |  31 +++
 v2v/var_expander.ml           |  69 +++++++
 v2v/var_expander.mli          |  82 ++++++++
 v2v/var_expander_tests.ml     | 103 ++++++++++
 v2v/virt-v2v-output-local.pod |  50 +++++
 v2v/virt-v2v.pod              |  15 +-
 17 files changed, 937 insertions(+), 12 deletions(-)
 create mode 100644 v2v/create_json.ml
 create mode 100644 v2v/create_json.mli
 create mode 100644 v2v/dummy.c
 create mode 100644 v2v/output_json.ml
 create mode 100644 v2v/output_json.mli
 create mode 100644 v2v/var_expander.ml
 create mode 100644 v2v/var_expander.mli
 create mode 100644 v2v/var_expander_tests.ml

-- 
2.20.1

This way it is possible to change where the matching start, instead of
always assuming it is the beginning.
---
 common/mlpcre/PCRE.ml       |  2 +-
 common/mlpcre/PCRE.mli      |  5 ++++-
 common/mlpcre/pcre-c.c      | 16 +++++++++++++---
 common/mlpcre/pcre_tests.ml | 15 ++++++++++++---
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/common/mlpcre/PCRE.ml b/common/mlpcre/PCRE.ml
index b054928f9..33074af1c 100644
--- a/common/mlpcre/PCRE.ml
+++ b/common/mlpcre/PCRE.ml
@@ -23,7 +23,7 @@ exception Error of string * int
 type regexp
 
 external compile : ?anchored:bool -> ?caseless:bool -> ?dotall:bool ->
?extended:bool -> ?multiline:bool -> string -> regexp =
"guestfs_int_pcre_compile_byte" "guestfs_int_pcre_compile"
-external matches : regexp -> string -> bool =
"guestfs_int_pcre_matches"
+external matches : ?offset:int -> regexp -> string -> bool =
"guestfs_int_pcre_matches"
 external sub : int -> string = "guestfs_int_pcre_sub"
 external subi : int -> int * int = "guestfs_int_pcre_subi"
 
diff --git a/common/mlpcre/PCRE.mli b/common/mlpcre/PCRE.mli
index eacb6fd90..e10d512fc 100644
--- a/common/mlpcre/PCRE.mli
+++ b/common/mlpcre/PCRE.mli
@@ -62,7 +62,7 @@ val compile : ?anchored:bool -> ?caseless:bool ->
?dotall:bool -> ?extended:bool
     See pcreapi(3) for details of what they do.
     All flags default to false. *)
 
-val matches : regexp -> string -> bool
+val matches : ?offset:int -> regexp -> string -> bool
 (** Test whether the regular expression matches the string.  This
     returns true if the regexp matches or false otherwise.
 
@@ -71,6 +71,9 @@ val matches : regexp -> string -> bool
     or the thread/program exits.  You can call {!sub} to return
     these substrings.
 
+    The [?offset] flag is used to change the start of the search,
+    which by default is at the beginning of the string (position 0).
+
     This can raise {!Error} if PCRE returns an error. *)
 
 val sub : int -> string
diff --git a/common/mlpcre/pcre-c.c b/common/mlpcre/pcre-c.c
index 0762a8341..be054a004 100644
--- a/common/mlpcre/pcre-c.c
+++ b/common/mlpcre/pcre-c.c
@@ -121,6 +121,15 @@ is_Some_true (value v)
     Bool_val (Field (v, 0)) /* Some true */;
 }
 
+static int
+Optint_val (value intv, int defval)
+{
+  if (intv == Val_int (0))      /* None */
+    return defval;
+  else                          /* Some int */
+    return Int_val (Field (intv, 0));
+}
+
 value
 guestfs_int_pcre_compile (value anchoredv, value caselessv, value dotallv,
                           value extendedv, value multilinev,
@@ -165,9 +174,9 @@ guestfs_int_pcre_compile_byte (value *argv, int argn)
 }
 
 value
-guestfs_int_pcre_matches (value rev, value strv)
+guestfs_int_pcre_matches (value offsetv, value rev, value strv)
 {
-  CAMLparam2 (rev, strv);
+  CAMLparam3 (offsetv, rev, strv);
   pcre *re = Regexp_val (rev);
   struct last_match *m, *oldm;
   size_t len = caml_string_length (strv);
@@ -205,7 +214,8 @@ guestfs_int_pcre_matches (value rev, value strv)
     caml_raise_out_of_memory ();
   }
 
-  m->r = pcre_exec (re, NULL, m->subject, len, 0, 0, m->vec, veclen);
+  m->r = pcre_exec (re, NULL, m->subject, len, Optint_val (offsetv, 0),
0,
+                    m->vec, veclen);
   if (m->r < 0 && m->r != PCRE_ERROR_NOMATCH) {
     int ret = m->r;
     free_last_match (m);
diff --git a/common/mlpcre/pcre_tests.ml b/common/mlpcre/pcre_tests.ml
index 346019c40..f199ad63a 100644
--- a/common/mlpcre/pcre_tests.ml
+++ b/common/mlpcre/pcre_tests.ml
@@ -18,6 +18,10 @@
 
 open Printf
 
+let optget def = function
+  | None -> def
+  | Some v -> v
+
 let compile ?(anchored = false) ?(caseless = false)
             ?(dotall = false) ?(extended = false) ?(multiline = false)
             patt @@ -30,9 +34,9 @@ let compile ?(anchored = false) ?(caseless =
false)
           patt;
   PCRE.compile ~anchored ~caseless ~dotall ~extended ~multiline patt
 
-let matches re str -  eprintf "PCRE.matches %s ->%!" str;
-  let r = PCRE.matches re str in
+let matches ?offset re str +  eprintf "PCRE.matches %s, %d ->%!"
str (optget 0 offset);
+  let r = PCRE.matches ?offset re str in
   eprintf " %b\n%!" r;
   r
 
@@ -103,6 +107,11 @@ let ()      assert (subi 1 = (2, 3));
     assert (subi 2 = (3, 3));
 
+    assert (matches ~offset:5 re0 "aaabcabc" = true);
+    assert (sub 0 = "ab");
+
+    assert (matches ~offset:5 re0 "aaabcbaac" = false);
+
     assert (replace re0 "dd" "abcabcaabccca" =
"ddcabcaabccca");
     assert (replace ~global:true re0 "dd" "abcabcaabccca" =
"ddcddcddccca");
 
-- 
2.20.1

This helper module provides a facility to replace %{FOO}-like variables
in text strings with user-provided content.
---
 .gitignore                |   1 +
 v2v/Makefile.am           |  32 +++++++++++-
 v2v/dummy.c               |   2 +
 v2v/var_expander.ml       |  69 +++++++++++++++++++++++++
 v2v/var_expander.mli      |  82 ++++++++++++++++++++++++++++++
 v2v/var_expander_tests.ml | 103 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 287 insertions(+), 2 deletions(-)
 create mode 100644 v2v/dummy.c
 create mode 100644 v2v/var_expander.ml
 create mode 100644 v2v/var_expander.mli
 create mode 100644 v2v/var_expander_tests.ml

diff --git a/.gitignore b/.gitignore
index 29d3e3aae..9a448fc4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -694,6 +694,7 @@ Makefile.in
 /v2v/uefi.ml
 /v2v/uefi.mli
 /v2v/v2v_unit_tests
+/v2v/var_expander_tests
 /v2v/virt-v2v
 /v2v/virt-v2v.1
 /v2v/virt-v2v-copy-to-local
diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index 2312812fb..f196be81d 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -98,6 +98,7 @@ SOURCES_MLI = \
 	utils.mli \
 	v2v.mli \
 	vCenter.mli \
+	var_expander.mli \
 	windows.mli \
 	windows_virtio.mli
 
@@ -106,6 +107,7 @@ SOURCES_ML = \
 	types.ml \
 	uefi.ml \
 	utils.ml \
+	var_expander.ml \
 	python_script.ml \
 	name_from_disk.ml \
 	vCenter.ml \
@@ -442,7 +444,7 @@ TESTS += \
 endif
 
 if HAVE_OCAML_PKG_OUNIT
-TESTS += v2v_unit_tests
+TESTS += v2v_unit_tests var_expander_tests
 endif
 
 if ENABLE_APPLIANCE
@@ -651,7 +653,7 @@ EXTRA_DIST += \
 # Unit tests.
 check_PROGRAMS  if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += v2v_unit_tests
+check_PROGRAMS += v2v_unit_tests var_expander_tests
 endif
 
 v2v_unit_tests_BOBJECTS = \
@@ -671,13 +673,28 @@ v2v_unit_tests_SOURCES = $(virt_v2v_SOURCES)
 v2v_unit_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS)
 v2v_unit_tests_CFLAGS = $(virt_v2v_CFLAGS)
 
+var_expander_tests_BOBJECTS = \
+	var_expander.cmo \
+	var_expander_tests.cmo
+var_expander_tests_XOBJECTS = $(var_expander_tests_BOBJECTS:.cmo=.cmx)
+
+var_expander_tests_SOURCES = dummy.c
+var_expander_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS)
+var_expander_tests_CFLAGS = $(virt_v2v_CFLAGS)
+
 if !HAVE_OCAMLOPT
 # Can't call this v2v_unit_tests_OBJECTS because automake gets confused.
 v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_BOBJECTS)
 v2v_unit_tests.cmo: OCAMLPACKAGES += -package oUnit
+
+var_expander_tests_THEOBJECTS = $(var_expander_tests_BOBJECTS)
+var_expander_tests.cmo: OCAMLPACKAGES += -package oUnit
 else
 v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_XOBJECTS)
 v2v_unit_tests.cmx: OCAMLPACKAGES += -package oUnit
+
+var_expander_tests_THEOBJECTS = $(var_expander_tests_XOBJECTS)
+var_expander_tests.cmx: OCAMLPACKAGES += -package oUnit
 endif
 
 v2v_unit_tests_DEPENDENCIES = \
@@ -696,6 +713,17 @@ v2v_unit_tests_LINK = \
 	  $(OCAMLLINKFLAGS) \
 	  $(v2v_unit_tests_THEOBJECTS) -o $@
 
+var_expander_tests_DEPENDENCIES = \
+	$(var_expander_tests_THEOBJECTS) \
+	../common/mlpcre/mlpcre.$(MLARCHIVE) \
+	$(top_srcdir)/ocaml-link.sh
+var_expander_tests_LINK = \
+	$(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+	  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) \
+	  $(OCAMLPACKAGES) -package oUnit \
+	  $(OCAMLLINKFLAGS) \
+	  $(var_expander_tests_THEOBJECTS) -o $@
+
 # Dependencies.
 .depend: \
 	$(srcdir)/*.mli \
diff --git a/v2v/dummy.c b/v2v/dummy.c
new file mode 100644
index 000000000..ebab6198c
--- /dev/null
+++ b/v2v/dummy.c
@@ -0,0 +1,2 @@
+/* Dummy source, to be used for OCaml-based tools with no C sources. */
+enum { foo = 1 };
diff --git a/v2v/var_expander.ml b/v2v/var_expander.ml
new file mode 100644
index 000000000..c47b8822b
--- /dev/null
+++ b/v2v/var_expander.ml
@@ -0,0 +1,69 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+
+exception Invalid_variable of string
+
+let var_re = PCRE.compile "%{([^}]+)}"
+
+let check_variable var +  String.iter (
+    function
+    | '0'..'9'
+    | 'a'..'z'
+    | 'A'..'Z'
+    | '_'
+    | '-' -> ()
+    | _ -> raise (Invalid_variable var)
+  ) var
+
+let scan_variables str +  let res = ref [] in
+  let offset = ref 0 in
+  while PCRE.matches ~offset:!offset var_re str; do
+    let var = PCRE.sub 1 in
+    check_variable var;
+    let _, end_ = PCRE.subi 0 in
+    List.push_back res var;
+    offset := end_
+  done;
+  List.remove_duplicates !res
+
+let replace_fn str fn +  let res = ref str in
+  let offset = ref 0 in
+  while PCRE.matches ~offset:!offset var_re !res; do
+    let var = PCRE.sub 1 in
+    check_variable var;
+    let start_, end_ = PCRE.subi 0 in
+    match fn var with
+    | None ->
+      offset := end_
+    | Some text ->
+      res := (String.sub !res 0 start_) ^ text ^ (String.sub !res end_
(String.length !res - end_));
+      offset := start_ + String.length text
+  done;
+  !res
+
+let replace_list str lst +  let fn var +    try Some (List.assoc var lst)
+    with Not_found -> None
+  in
+  replace_fn str fn
diff --git a/v2v/var_expander.mli b/v2v/var_expander.mli
new file mode 100644
index 000000000..80aa33c2c
--- /dev/null
+++ b/v2v/var_expander.mli
@@ -0,0 +1,82 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Simple variable expander.
+
+    This module provides the support to expand variables in strings,
+    specified in the form of [%{name}].
+
+    For example:
+
+{v
+let str = "variable-%{INDEX} in %{INDEX} replaced %{INDEX} times"
+let index = ref 0
+let fn = function
+  | "INDEX" ->
+    incr index;
+    Some (string_of_int !index)
+  | _ -> None
+in
+let str = Var_expander.replace_fn str fn
+(* now str is "variable-1 in 2 replaced 3 times" *)
+v}
+
+    The names of variables can contain only ASCII letters (uppercase,
+    and lowercase), digits, underscores, and dashes.
+
+    The replacement is done in a single pass: this means that if a
+    variable is replaced with the text of a variable, that new text
+    is kept as is in the final output.  In practice:
+
+{v
+let str = "%{VAR}"
+let str = Var_expander.replace_list str [("VAR", "%{VAR}")]
+(* now str is "%{VAR}" *)
+v}
+*)
+
+exception Invalid_variable of string
+(** Invalid variable name error.
+
+    In case a variable contains characters not allowed, then this
+    exception with the actual unacceptable variable. *)
+
+val scan_variables : string -> string list
+(** Scan the pattern string for all the variables available.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)
+
+val replace_fn : string -> (string -> string option) -> string
+(** Replaces a string expanding all the variables.
+
+    The replacement function specify how a variable is replaced;
+    if [None] is returned, then that variable is not replaced.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)
+
+val replace_list : string -> (string * string) list -> string
+(** Replaces a string expanding all the variables.
+
+    The replacement list specify how a variable is replaced;
+    if it is not specified in the list, then that variable is not
+    replaced.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)
diff --git a/v2v/var_expander_tests.ml b/v2v/var_expander_tests.ml
new file mode 100644
index 000000000..d9dec9e7c
--- /dev/null
+++ b/v2v/var_expander_tests.ml
@@ -0,0 +1,103 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open OUnit
+
+let assert_equal_string = assert_equal ~printer:(fun x -> x)
+let assert_equal_stringlist = assert_equal ~printer:(fun x -> "("
^ (String.escaped (String.concat "," x)) ^ ")")
+
+let replace_none_fn _ = None
+let replace_empty_fn _ = Some ""
+
+let test_no_replacement () +  assert_equal_string ""
(Var_expander.replace_fn "" replace_none_fn);
+  assert_equal_string "x" (Var_expander.replace_fn "x"
replace_none_fn);
+  assert_equal_string "%{}" (Var_expander.replace_fn "%{}"
replace_none_fn);
+  assert_equal_string "%{EMPTY}" (Var_expander.replace_fn
"%{EMPTY}" replace_none_fn);
+  assert_equal_string "%{EMPTY} %{no}" (Var_expander.replace_fn
"%{EMPTY} %{no}" replace_none_fn);
+  assert_equal_string "a %{EMPTY} b" (Var_expander.replace_fn "a
%{EMPTY} b" replace_none_fn);
+  ()
+
+let test_replacements () +  assert_equal_string ""
(Var_expander.replace_fn "%{EMPTY}" replace_empty_fn);
+  assert_equal_string "x " (Var_expander.replace_fn "x
%{EMPTY}" replace_empty_fn);
+  assert_equal_string "xy" (Var_expander.replace_fn
"x%{EMPTY}y" replace_empty_fn);
+  assert_equal_string "x<->y" (Var_expander.replace_fn
"x%{FOO}y" (function | "FOO" -> Some
"<->" | _ -> None));
+  assert_equal_string "a x b" (Var_expander.replace_fn "a %{FOO}
b" (function | "FOO" -> Some "x" | _ -> None));
+  assert_equal_string "%{FOO} x" (Var_expander.replace_fn
"%{FOO} %{BAR}" (function | "BAR" -> Some "x" |
_ -> None));
+  assert_equal_string "%{FOO}" (Var_expander.replace_fn
"%{BAR}" (function | "BAR" -> Some "%{FOO}" | _
-> None));
+  assert_equal_string "%{FOO} x" (Var_expander.replace_fn
"%{BAR} %{FOO}" (function | "BAR" -> Some
"%{FOO}" | "FOO" -> Some "x" | _ -> None));
+  begin
+    let str = "%{INDEX}, %{INDEX}, %{INDEX}" in
+    let index = ref 0 in
+    let fn = function
+      | "INDEX" ->
+        incr index;
+        Some (string_of_int !index)
+      | _ -> None
+    in
+    assert_equal_string "1, 2, 3" (Var_expander.replace_fn str fn)
+  end;
+  ()
+
+let test_list () +  assert_equal_string "x %{NONE}"
(Var_expander.replace_list "%{FOO} %{NONE}" [("FOO",
"x")]);
+  ()
+
+let test_scan_variables () +  let assert_invalid_variable var +    let str =
"%{" ^ var ^ "}" in
+    assert_raises (Var_expander.Invalid_variable var)
+                  (fun () -> Var_expander.scan_variables str)
+  in
+  assert_equal_stringlist [] (Var_expander.scan_variables "");
+  assert_equal_stringlist [] (Var_expander.scan_variables "foo");
+  assert_equal_stringlist ["FOO"] (Var_expander.scan_variables
"%{FOO}");
+  assert_equal_stringlist ["FOO"; "BAR"]
(Var_expander.scan_variables "%{FOO} %{BAR}");
+  assert_equal_stringlist ["FOO"; "BAR"]
(Var_expander.scan_variables "%{FOO} %{BAR} %{FOO}");
+  assert_invalid_variable "FOO/BAR";
+  ()
+
+let test_errors () +  let assert_invalid_variable var +    let str =
"%{" ^ var ^ "}" in
+    assert_raises (Var_expander.Invalid_variable var)
+                  (fun () -> Var_expander.replace_fn str replace_none_fn)
+  in
+  assert_invalid_variable "FOO/BAR";
+  assert_invalid_variable "FOO:BAR";
+  assert_invalid_variable "FOO(BAR";
+  assert_invalid_variable "FOO)BAR";
+  assert_invalid_variable "FOO@BAR";
+  ()
+
+(* Suites declaration. *)
+let suite +  TestList ([
+    "basic" >::: [
+      "no_replacement" >:: test_no_replacement;
+      "replacements" >:: test_replacements;
+      "list" >:: test_list;
+      "scan_variables" >:: test_scan_variables;
+      "errors" >:: test_errors;
+    ];
+  ])
+
+let () +  ignore (run_test_tt_main suite);
+  Printf.fprintf stderr "\n"
-- 
2.20.1

Add a new output mode to virt-v2v: similar to -o local, the written
metadata is a JSON file with the majority of the data that virt-v2v
knowns about (or collects) during the conversion.

This is meant to be used only when no existing output mode is usable,
and a guest needs to be converted to run on KVM anyway.  The user of
this mode is supposed to use all the data in the JSON, as they contain
important details on how even run the guest (e.g. w.r.t. firmware,
drivers of disks/NICs, etc).
---
 v2v/Makefile.am               |   4 +
 v2v/cmdline.ml                |  29 +++
 v2v/create_json.ml            | 348 ++++++++++++++++++++++++++++++++++
 v2v/create_json.mli           |  29 +++
 v2v/output_json.ml            | 116 ++++++++++++
 v2v/output_json.mli           |  31 +++
 v2v/virt-v2v-output-local.pod |  50 +++++
 v2v/virt-v2v.pod              |  15 +-
 8 files changed, 620 insertions(+), 2 deletions(-)
 create mode 100644 v2v/create_json.ml
 create mode 100644 v2v/create_json.mli
 create mode 100644 v2v/output_json.ml
 create mode 100644 v2v/output_json.mli

diff --git a/v2v/Makefile.am b/v2v/Makefile.am
index f196be81d..53c137fc6 100644
--- a/v2v/Makefile.am
+++ b/v2v/Makefile.am
@@ -52,6 +52,7 @@ SOURCES_MLI = \
 	config.mli \
 	convert_linux.mli \
 	convert_windows.mli \
+	create_json.mli \
 	create_libvirt_xml.mli \
 	create_ovf.mli \
 	DOM.mli \
@@ -75,6 +76,7 @@ SOURCES_MLI = \
 	networks.mli \
 	openstack_image_properties.mli \
 	output_glance.mli \
+	output_json.mli \
 	output_libvirt.mli \
 	output_local.mli \
 	output_null.mli \
@@ -117,6 +119,7 @@ SOURCES_ML = \
 	parse_ovf_from_ova.ml \
 	parse_ova.ml \
 	create_ovf.ml \
+	create_json.ml \
 	linux.ml \
 	windows.ml \
 	windows_virtio.ml \
@@ -141,6 +144,7 @@ SOURCES_ML = \
 	convert_windows.ml \
 	output_null.ml \
 	output_glance.ml \
+	output_json.ml \
 	output_libvirt.ml \
 	output_local.ml \
 	output_qemu.ml \
diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
index 46f6910d0..4d390f249 100644
--- a/v2v/cmdline.ml
+++ b/v2v/cmdline.ml
@@ -138,6 +138,7 @@ let parse_cmdline ()      | "glance" ->
output_mode := `Glance
     | "libvirt" -> output_mode := `Libvirt
     | "disk" | "local" -> output_mode := `Local
+    | "json" -> output_mode := `JSON
     | "null" -> output_mode := `Null
     | "openstack" | "osp" | "rhosp" ->
output_mode := `Openstack
     | "ovirt" | "rhv" | "rhev" -> output_mode
:= `RHV
@@ -413,6 +414,17 @@ read the man page virt-v2v(1).
     | `RHV -> no_options (); `RHV
     | `QEmu -> no_options (); `QEmu
 
+    | `JSON ->
+       if is_query then (
+         Output_json.print_output_options ();
+         exit 0
+       )
+       else (
+         let json_options +           Output_json.parse_output_options
output_options in
+         `JSON json_options
+       )
+
     | `Openstack ->
        if is_query then (
          Output_openstack.print_output_options ();
@@ -546,6 +558,23 @@ read the man page virt-v2v(1).
       Output_libvirt.output_libvirt output_conn output_storage,
       output_format, output_alloc
 
+    | `JSON json_options ->
+      if output_password <> None then
+        error_option_cannot_be_used_in_output_mode "json"
"-op";
+      if output_conn <> None then
+        error_option_cannot_be_used_in_output_mode "json"
"-oc";
+      let os +        match output_storage with
+        | None ->
+           error (f_"-o json: output directory was not specified, use
'-os /dir'")
+        | Some d when not (is_directory d) ->
+           error (f_"-os %s: output directory does not exist or is not a
directory") d
+        | Some d -> d in
+      if qemu_boot then
+        error_option_cannot_be_used_in_output_mode "json"
"--qemu-boot";
+      Output_json.output_json os json_options,
+      output_format, output_alloc
+
     | `Local ->
       if output_password <> None then
         error_option_cannot_be_used_in_output_mode "local"
"-op";
diff --git a/v2v/create_json.ml b/v2v/create_json.ml
new file mode 100644
index 000000000..fdf7b12f5
--- /dev/null
+++ b/v2v/create_json.ml
@@ -0,0 +1,348 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open C_utils
+open Tools_utils
+
+open Types
+open Utils
+
+module G = Guestfs
+
+let json_list_of_string_list +  List.map (fun x -> JSON.String x)
+
+let json_list_of_string_string_list +  List.map (fun (x, y) -> x,
JSON.String y)
+
+let push_optional_string lst name = function
+  | None -> ()
+  | Some v -> List.push_back lst (name, JSON.String v)
+
+let push_optional_int lst name = function
+  | None -> ()
+  | Some v -> List.push_back lst (name, JSON.Int (Int64.of_int v))
+
+let json_unknown_string = function
+  | "unknown" -> JSON.Null
+  | v -> JSON.String v
+
+let find_target_disk targets { s_disk_id = id } +  try List.find (fun t ->
t.target_overlay.ov_source.s_disk_id = id) targets
+  with Not_found -> assert false
+
+let create_json_metadata source targets target_buses
+                         guestcaps inspect target_firmware +  let doc = ref [
+    "version", JSON.Int 1L;
+    "name", JSON.String source.s_name;
+    "memory", JSON.Int source.s_memory;
+    "vcpu", JSON.Int (Int64.of_int source.s_vcpu);
+  ] in
+
+  (match source.s_genid with
+   | None -> ()
+   | Some genid -> List.push_back doc ("genid", JSON.String genid)
+  );
+
+  if source.s_cpu_vendor <> None || source.s_cpu_model <> None ||
+     source.s_cpu_topology <> None then (
+    let cpu = ref [] in
+
+    push_optional_string cpu "vendor" source.s_cpu_vendor;
+    push_optional_string cpu "model" source.s_cpu_model;
+    (match source.s_cpu_topology with
+     | None -> ()
+     | Some { s_cpu_sockets; s_cpu_cores; s_cpu_threads } ->
+        let attrs = [
+          "sockets", JSON.Int (Int64.of_int s_cpu_sockets);
+          "cores", JSON.Int (Int64.of_int s_cpu_cores);
+          "threads", JSON.Int (Int64.of_int s_cpu_threads);
+        ] in
+        List.push_back cpu ("topology", JSON.Dict attrs)
+    );
+
+    List.push_back doc ("cpu", JSON.Dict !cpu);
+  );
+
+  let firmware +    let firmware_type +      match target_firmware with
+      | TargetBIOS -> "bios"
+      | TargetUEFI -> "uefi" in
+
+    let fw = ref [
+      "type", JSON.String firmware_type;
+    ] in
+
+    (match target_firmware with
+     | TargetBIOS -> ()
+     | TargetUEFI ->
+       let uefi_firmware = find_uefi_firmware guestcaps.gcaps_arch in
+       let flags +         List.map (
+           function
+           | Uefi.UEFI_FLAG_SECURE_BOOT_REQUIRED ->
"secure_boot_required"
+         ) uefi_firmware.Uefi.flags in
+
+       let uefi = ref [
+         "code", JSON.String uefi_firmware.Uefi.code;
+         "vars", JSON.String uefi_firmware.Uefi.vars;
+         "flags", JSON.List (json_list_of_string_list flags);
+       ] in
+
+       push_optional_string uefi "code-debug"
uefi_firmware.Uefi.code_debug;
+
+       List.push_back fw ("uefi", JSON.Dict !uefi)
+    );
+
+    !fw in
+  List.push_back doc ("firmware", JSON.Dict firmware);
+
+  List.push_back doc ("features",
+                      JSON.List (json_list_of_string_list source.s_features));
+
+  let machine +    match guestcaps.gcaps_machine with
+    | I440FX -> "pc"
+    | Q35 -> "q35"
+    | Virt -> "virt" in
+  List.push_back doc ("machine", JSON.String machine);
+
+  let disks, removables +    let disks = ref []
+    and removables = ref [] in
+
+    let iter_bus bus_name drive_prefix i = function
+    | BusSlotEmpty -> ()
+    | BusSlotDisk d ->
+       (* Find the corresponding target disk. *)
+       let t = find_target_disk targets d in
+
+       let target_file +         match t.target_file with
+         | TargetFile s -> s
+         | TargetURI _ -> assert false in
+
+       let disk = [
+         "dev", JSON.String (drive_prefix ^ drive_name i);
+         "bus", JSON.String bus_name;
+         "format", JSON.String t.target_format;
+         "file", JSON.String (absolute_path target_file);
+       ] in
+
+       List.push_back disks (JSON.Dict disk)
+
+    | BusSlotRemovable { s_removable_type = CDROM } ->
+       let cdrom = [
+         "type", JSON.String "cdrom";
+         "dev", JSON.String (drive_prefix ^ drive_name i);
+         "bus", JSON.String bus_name;
+       ] in
+
+       List.push_back removables (JSON.Dict cdrom)
+
+    | BusSlotRemovable { s_removable_type = Floppy } ->
+       let floppy = [
+         "type", JSON.String "floppy";
+         "dev", JSON.String (drive_prefix ^ drive_name i);
+       ] in
+
+       List.push_back removables (JSON.Dict floppy)
+    in
+
+    Array.iteri (iter_bus "virtio" "vd")
target_buses.target_virtio_blk_bus;
+    Array.iteri (iter_bus "ide" "hd")
target_buses.target_ide_bus;
+    Array.iteri (iter_bus "scsi" "sd")
target_buses.target_scsi_bus;
+    Array.iteri (iter_bus "floppy" "fd")
target_buses.target_floppy_bus;
+
+    !disks, !removables in
+  List.push_back doc ("disks", JSON.List disks);
+  List.push_back doc ("removables", JSON.List removables);
+
+  let nics +    List.map (
+      fun { s_mac = mac; s_vnet_type = vnet_type; s_nic_model = nic_model;
+            s_vnet = vnet; } ->
+        let vnet_type_str +          match vnet_type with
+          | Bridge -> "bridge"
+          | Network -> "network" in
+
+        let nic = ref [
+          "vnet", JSON.String vnet;
+          "vnet-type", JSON.String vnet_type_str;
+        ] in
+
+        let nic_model_str = Option.map string_of_nic_model nic_model in
+        push_optional_string nic "model" nic_model_str;
+
+        push_optional_string nic "mac" mac;
+
+        JSON.Dict !nic
+    ) source.s_nics in
+  List.push_back doc ("nics", JSON.List nics);
+
+  let guestcaps_dict +    let block_bus +      match guestcaps.gcaps_block_bus
with
+      | Virtio_blk -> "virtio-blk"
+      | Virtio_SCSI -> "virtio-scsi"
+      | IDE -> "ide" in
+    let net_bus +      match guestcaps.gcaps_net_bus with
+      | Virtio_net -> "virtio-net"
+      | E1000 -> "e1000"
+      | RTL8139 -> "rtl8139" in
+    let video +      match guestcaps.gcaps_video with
+      | QXL -> "qxl"
+      | Cirrus -> "cirrus" in
+    let machine +      match guestcaps.gcaps_machine with
+      | I440FX -> "i440fx"
+      | Q35 -> "q35"
+      | Virt -> "virt" in
+
+    [
+      "block-bus", JSON.String block_bus;
+      "net-bus", JSON.String net_bus;
+      "video", JSON.String video;
+      "machine", JSON.String machine;
+      "arch", JSON.String guestcaps.gcaps_arch;
+      "virtio-rng", JSON.Bool guestcaps.gcaps_virtio_rng;
+      "virtio-balloon", JSON.Bool guestcaps.gcaps_virtio_balloon;
+      "isa-pvpanic", JSON.Bool guestcaps.gcaps_isa_pvpanic;
+      "acpi", JSON.Bool guestcaps.gcaps_acpi;
+    ] in
+  List.push_back doc ("guestcaps", JSON.Dict guestcaps_dict);
+
+  (match source.s_sound with
+   | None -> ()
+   | Some { s_sound_model = model } ->
+     let sound = [
+       "model", JSON.String (string_of_source_sound_model model);
+     ] in
+     List.push_back doc ("sound", JSON.Dict sound)
+   );
+
+  (match source.s_display with
+   | None -> ()
+   | Some d ->
+     let display_type +       match d.s_display_type with
+       | Window -> "window"
+       | VNC -> "vnc"
+       | Spice -> "spice" in
+
+     let display = ref [
+       "type", JSON.String display_type;
+     ] in
+
+     push_optional_string display "keymap" d.s_keymap;
+     push_optional_string display "password" d.s_password;
+
+     let listen +       match d.s_listen with
+       | LNoListen -> None
+       | LAddress address ->
+         Some [
+           "type", JSON.String "address";
+           "address", JSON.String address;
+         ]
+       | LNetwork network ->
+         Some [
+           "type", JSON.String "network";
+           "network", JSON.String network;
+         ]
+       | LSocket None ->
+         Some [
+           "type", JSON.String "socket";
+           "socket", JSON.Null;
+         ]
+       | LSocket (Some socket) ->
+         Some [
+           "type", JSON.String "socket";
+           "socket", JSON.String socket;
+         ]
+       | LNone ->
+         Some [
+           "type", JSON.String "none";
+         ] in
+     (match listen with
+      | None -> ()
+      | Some l -> List.push_back display ("listen", JSON.Dict l)
+     );
+
+     push_optional_int display "port" d.s_port;
+
+     List.push_back doc ("display", JSON.Dict !display)
+  );
+
+  let inspect_dict +    let apps +      List.map (
+        fun { G.app2_name = name; app2_display_name = display_name;
+              app2_epoch = epoch; app2_version = version;
+              app2_release = release; app2_arch = arch; } ->
+          JSON.Dict [
+            "name", JSON.String name;
+            "display-name", JSON.String display_name;
+            "epoch", JSON.Int (Int64.of_int32 epoch);
+            "version", JSON.String version;
+            "release", JSON.String release;
+            "arch", JSON.String arch;
+          ]
+      ) inspect.i_apps in
+
+    let firmware_dict +      match inspect.i_firmware with
+      | I_BIOS ->
+        [
+          "type", JSON.String "bios";
+        ]
+      | I_UEFI devices ->
+        [
+          "type", JSON.String "uefi";
+          "devices", JSON.List (json_list_of_string_list devices);
+        ] in
+
+    [
+      "root", JSON.String inspect.i_root;
+      "type", JSON.String inspect.i_type;
+      "distro", json_unknown_string inspect.i_distro;
+      "osinfo", json_unknown_string inspect.i_osinfo;
+      "arch", JSON.String inspect.i_arch;
+      "major-version", JSON.Int (Int64.of_int
inspect.i_major_version);
+      "minor-version", JSON.Int (Int64.of_int
inspect.i_minor_version);
+      "package-format", json_unknown_string inspect.i_package_format;
+      "package-management", json_unknown_string
inspect.i_package_management;
+      "product-name", json_unknown_string inspect.i_product_name;
+      "product-variant", json_unknown_string
inspect.i_product_variant;
+      "mountpoints", JSON.Dict (json_list_of_string_string_list
inspect.i_mountpoints);
+      "applications", JSON.List apps;
+      "windows-systemroot", JSON.String inspect.i_windows_systemroot;
+      "windows-software-hive", JSON.String
inspect.i_windows_software_hive;
+      "windows-system-hive", JSON.String
inspect.i_windows_system_hive;
+      "windows-current-control-set", JSON.String
inspect.i_windows_current_control_set;
+      "firmware", JSON.Dict firmware_dict;
+    ] in
+  List.push_back doc ("inspect", JSON.Dict inspect_dict);
+
+  !doc
diff --git a/v2v/create_json.mli b/v2v/create_json.mli
new file mode 100644
index 000000000..6dbb6e48b
--- /dev/null
+++ b/v2v/create_json.mli
@@ -0,0 +1,29 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Create JSON metadata for [-o json]. *)
+
+val create_json_metadata : Types.source -> Types.target list ->
+                           Types.target_buses ->
+                           Types.guestcaps ->
+                           Types.inspect ->
+                           Types.target_firmware ->
+                           JSON.doc
+(** [create_json_metadata source targets target_buses guestcaps
+    inspect target_firmware] creates the JSON with the majority
+    of the data that virt-v2v used for the conversion. *)
diff --git a/v2v/output_json.ml b/v2v/output_json.ml
new file mode 100644
index 000000000..ca0bda978
--- /dev/null
+++ b/v2v/output_json.ml
@@ -0,0 +1,116 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Printf
+
+open Std_utils
+open Tools_utils
+open Common_gettext.Gettext
+
+open Types
+open Utils
+
+type json_options = {
+  json_disks_pattern : string;
+}
+
+let print_output_options () +  printf (f_"Output options (-oo) which can
be used with -o json:
+
+  -oo json-disks-pattern=PATTERN   Pattern for the disks.
+")
+
+let known_pattern_variables = ["DiskNo"; "DiskDeviceName";
"GuestName"]
+
+let parse_output_options options +  let json_disks_pattern = ref None in
+
+  List.iter (
+    function
+    | "json-disks-pattern", v ->
+       if !json_disks_pattern <> None then
+         error (f_"-o json: -oo json-disks-pattern set more than
once");
+       let vars +         try Var_expander.scan_variables v
+         with Var_expander.Invalid_variable var ->
+           error (f_"-o json: -oo json-disks-pattern: invalid variable
%%{%s}")
+             var in
+       List.iter (
+         fun var ->
+           if not (List.mem var known_pattern_variables) then
+             error (f_"-o json: -oo json-disks-pattern: unhandled variable
%%{%s}")
+               var
+       ) vars;
+       json_disks_pattern := Some v
+    | k, _ ->
+       error (f_"-o json: unknown output option ‘-oo %s’") k
+  ) options;
+
+  let json_disks_pattern +    Option.default
"%{GuestName}-%{DiskDeviceName}" !json_disks_pattern in
+
+  { json_disks_pattern }
+
+class output_json dir json_options = object
+  inherit output
+
+  method as_options = sprintf "-o json -os %s" dir
+
+  method prepare_targets source overlays _ _ _ _ +    List.mapi (
+      fun i (_, ov) ->
+        let outname +          let vars_fn = function
+            | "DiskNo" -> Some (string_of_int (i+1))
+            | "DiskDeviceName" -> Some ov.ov_sd
+            | "GuestName" -> Some source.s_name
+            | _ -> assert false
+          in
+          Var_expander.replace_fn json_options.json_disks_pattern vars_fn in
+        let destname = dir // outname in
+        mkdir_p (Filename.dirname destname) 0o755;
+        TargetFile destname
+    ) overlays
+
+  method supported_firmware = [ TargetBIOS; TargetUEFI ]
+
+  method create_metadata source targets
+                         target_buses guestcaps inspect target_firmware +   
let doc +      Create_json.create_json_metadata source targets target_buses
+                                       guestcaps inspect target_firmware in
+    let doc_string = JSON.string_of_doc ~fmt:JSON.Indented doc in
+
+    if verbose () then (
+      eprintf "resulting JSON:\n";
+      output_string stderr doc_string;
+      eprintf "\n\n%!";
+    );
+
+    let name = source.s_name in
+    let file = dir // name ^ ".json" in
+
+    with_open_out file (
+      fun chan ->
+        output_string chan doc_string;
+        output_char chan '\n'
+    )
+end
+
+let output_json = new output_json
+let () = Modules_list.register_output_module "json"
diff --git a/v2v/output_json.mli b/v2v/output_json.mli
new file mode 100644
index 000000000..52f58f2d1
--- /dev/null
+++ b/v2v/output_json.mli
@@ -0,0 +1,31 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** [-o json] target. *)
+
+type json_options
+(** Miscellaneous extra command line parameters used by json. *)
+
+val print_output_options : unit -> unit
+val parse_output_options : (string * string) list -> json_options
+(** Print and parse json -oo options. *)
+
+val output_json : string -> json_options -> Types.output
+(** [output_json directory json_options] creates and returns a new
+    {!Types.output} object specialized for writing output to local
+    files with JSON metadata. *)
diff --git a/v2v/virt-v2v-output-local.pod b/v2v/virt-v2v-output-local.pod
index 7427b1ed7..ec737e1ba 100644
--- a/v2v/virt-v2v-output-local.pod
+++ b/v2v/virt-v2v-output-local.pod
@@ -11,6 +11,9 @@ or libvirt
 
  virt-v2v [-i* options] -o qemu -os DIRECTORY [--qemu-boot]
 
+ virt-v2v [-i* options] -o json -os DIRECTORY
+                        [-oo json-disks-pattern=PATTERN]
+
  virt-v2v [-i* options] -o null
 
 =head1 DESCRIPTION
@@ -54,6 +57,13 @@ above, a shell script is created which contains the raw qemu
command
 you would need to boot the guest.  However the shell script is not
 run, I<unless> you also add the I<--qemu-boot> option.
 
+=item B<-o json -os> C<DIRECTORY>
+
+This converts the guest to files in C<DIRECTORY>.  The metadata
+produced is a JSON file containing the majority of the data virt-v2v
+gathers during the conversion.
+See L</OUTPUT TO JSON> below.
+
 =item B<-o null>
 
 The guest is converted, but the final result is thrown away and no
@@ -140,6 +150,46 @@ Define the final guest in libvirt:
 
 =back
 
+=head1 OUTPUT TO JSON
+
+The I<-o json> option produces the following files by default:
+
+ NAME.json                     JSON metadata.
+ NAME-sda, NAME-sdb, etc.      Guest disk(s).
+
+where C<NAME> is the guest name.
+
+It is possible to change the pattern of the disks using the
+I<-oo json-disks-pattern=...> option: it allows parameters in form of
+C<%{...}> variables, for example:
+
+ -oo json-disks-pattern=disk%{DiskNo}.img
+
+Recognized variables are:
+
+=over 4
+
+=item C<%{DiskNo}>
+
+The index of the disk, starting from 1.
+
+=item C<%{DiskDeviceName}>
+
+The destination device of the disk, e.g. C<sda>, C<sdb>, etc.
+
+=item C<%{GuestName}>
+
+The name of the guest.
+
+=back
+
+Using a pattern it is possible use subdirectories for the disks,
+even with names depending on variables; for example:
+
+ -oo json-disks-pattern=%{GuestName}-%{DiskNo}/disk.img
+
+The default pattern is C<%{GuestName}-%{DiskDeviceName}>.
+
 =head1 SEE ALSO
 
 L<virt-v2v(1)>.
diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
index cf9464834..9a555c3be 100644
--- a/v2v/virt-v2v.pod
+++ b/v2v/virt-v2v.pod
@@ -425,6 +425,17 @@ instead.
 Set the output method to OpenStack Glance.  In this mode the converted
 guest is uploaded to Glance.  See L<virt-v2v-output-openstack(1)>.
 
+=item B<-o> B<json>
+
+Set the output method to I<json>.
+
+In this mode, the converted guest is written to a local directory
+specified by I<-os /dir> (the directory must exist), with a JSON file
+containing the majority of the metadata that virt-v2v gathered during
+the conversion.
+
+See L<virt-v2v-output-local(1)>.
+
 =item B<-o> B<libvirt>
 
 Set the output method to I<libvirt>.  This is the default.
@@ -696,8 +707,8 @@ The location of the storage for the converted guest.
 For I<-o libvirt>, this is a libvirt directory pool
 (see S<C<virsh pool-list>>) or pool UUID.
 
-For I<-o local> and I<-o qemu>, this is a directory name.  The
-directory must exist.
+For I<-o json>, I<-o local> and I<-o qemu>, this is a
directory name.
+The directory must exist.
 
 For I<-o rhv-upload>, this is the name of the destination Storage
 Domain.
-- 
2.20.1

On Mon, 25 Feb 2019 17:22:49 +0100
Pino Toscano <ptoscano@redhat.com> wrote:
> This series adds a new output mode for virt-v2v, called -o json.
> It produces local files, just like -o local, although the metadata
> produced is a JSON file with data that v2v collected in the conversion
> process.  This can be useful for converting to unsupported destinations,
> still based on QEMU/KVM.
> 
> In addition to a simple different metadata, it offers a way to relocate
> the disks, with %{...}-like variables (only 3 added ATM, more can be
> added) to change their paths depending on data of the guest/disks.
> 
> Thanks,
> 
> 
> Pino Toscano (3):
>   common/mlpcre: add offset flag for PCRE.matches
>   v2v: add Var_expander
>   v2v: add -o json output mode
> 
>  .gitignore                    |   1 +
>  common/mlpcre/PCRE.ml         |   2 +-
>  common/mlpcre/PCRE.mli        |   5 +-
>  common/mlpcre/pcre-c.c        |  16 +-
>  common/mlpcre/pcre_tests.ml   |  15 +-
>  v2v/Makefile.am               |  36 +++-
>  v2v/cmdline.ml                |  29 +++
>  v2v/create_json.ml            | 348 ++++++++++++++++++++++++++++++++++
>  v2v/create_json.mli           |  29 +++
>  v2v/dummy.c                   |   2 +
>  v2v/output_json.ml            | 116 ++++++++++++
>  v2v/output_json.mli           |  31 +++
>  v2v/var_expander.ml           |  69 +++++++
>  v2v/var_expander.mli          |  82 ++++++++
>  v2v/var_expander_tests.ml     | 103 ++++++++++
>  v2v/virt-v2v-output-local.pod |  50 +++++
>  v2v/virt-v2v.pod              |  15 +-
>  17 files changed, 937 insertions(+), 12 deletions(-)
>  create mode 100644 v2v/create_json.ml
>  create mode 100644 v2v/create_json.mli
>  create mode 100644 v2v/dummy.c
>  create mode 100644 v2v/output_json.ml
>  create mode 100644 v2v/output_json.mli
>  create mode 100644 v2v/var_expander.ml
>  create mode 100644 v2v/var_expander.mli
>  create mode 100644 v2v/var_expander_tests.ml
> 
Series LGTM

On Mon, Feb 25, 2019 at 05:22:50PM +0100, Pino Toscano
wrote:
> diff --git a/common/mlpcre/pcre_tests.ml b/common/mlpcre/pcre_tests.ml
> index 346019c40..f199ad63a 100644
> --- a/common/mlpcre/pcre_tests.ml
> +++ b/common/mlpcre/pcre_tests.ml
> @@ -18,6 +18,10 @@
>  
>  open Printf
>  
> +let optget def = function
> +  | None -> def
> +  | Some v -> v
This is the same as Option.default (from Std_utils), so this function
isn't needed.  But in any case OCaml will do this automatically for
you:
> +let matches ?offset re str 
  let matches ?(offset = 0) re str 
will replace offset by 0 if the labelled parameter wasn't given.
> +  eprintf "PCRE.matches %s, %d ->%!" str (optget 0 offset);
> +  let r = PCRE.matches ?offset re str in
With the change above, the type of offset is int so you can
pass it to PCRE.matches using ~offset instead of ?offset.

However in general this patch is fine, so ACK if you fix the above.

Rich.
>    eprintf " %b\n%!" r;
>    r
>  
> @@ -103,6 +107,11 @@ let () >      assert (subi 1 = (2, 3));
>      assert (subi 2 = (3, 3));
>  
> +    assert (matches ~offset:5 re0 "aaabcabc" = true);
> +    assert (sub 0 = "ab");
> +
> +    assert (matches ~offset:5 re0 "aaabcbaac" = false);
> +
>      assert (replace re0 "dd" "abcabcaabccca" =
"ddcabcaabccca");
>      assert (replace ~global:true re0 "dd"
"abcabcaabccca" = "ddcddcddccca");
>  
> -- 
> 2.20.1
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs
-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine.  Supports Linux and Windows.
http://people.redhat.com/~rjones/virt-df/

On Mon, Feb 25, 2019 at 05:22:51PM +0100, Pino Toscano wrote:
[...]

After being burned a few times with custom parsing (hello, guestfish)
I'm not a big fan.

Is there not an existing C or OCaml library/facility we could use
here?  It's a shame we can't use Perl Template Toolkit because it
would be ideal here.

There are all kinds of questions that aren't answered such as: Should
variables be replaced recursively?  How do you escape %{..} if you
don't want it to be replaced?  Should we allow loops or similar
constructs?  Existing template systems solve these kinds of problems
already.

Anyway ...
> +let var_re = PCRE.compile "%{([^}]+)}"
Are we planning to allow a completely free choice for variable names,
or could we limit this regexp to only matching ASCII alphanumeric +
underscore?
> +let check_variable var > +  String.iter (
> +    function
> +    | '0'..'9'
> +    | 'a'..'z'
> +    | 'A'..'Z'
> +    | '_'
> +    | '-' -> ()
> +    | _ -> raise (Invalid_variable var)
> +  ) var
... and then this function would presumably go away.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
Fedora Windows cross-compiler. Compile Windows programs, test, and
build Windows installers. Over 100 libraries supported.
http://fedoraproject.org/wiki/MinGW

If we pushed the baseline of OCaml up by (I think) just a single
version then most of this code could be generated automatically from
the description in the Types module.  It would rely on the "new"
(actually rather old) feature called extension points (ppx) which I
think was added in 4.02.

However in its own terms the idea behind this patch is fine.  As I
said in the previous email I'm somewhat unhappy with a homebrew
templating system - could you check if there's anything out there that
we could easily use already?  If not then I guess we'd need to go with
homebrew.

Rich.
> +  !doc
> diff --git a/v2v/create_json.mli b/v2v/create_json.mli
> new file mode 100644
> index 000000000..6dbb6e48b
> --- /dev/null
> +++ b/v2v/create_json.mli
> @@ -0,0 +1,29 @@
> +(* virt-v2v
> + * Copyright (C) 2019 Red Hat Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + *)
> +
> +(** Create JSON metadata for [-o json]. *)
> +
> +val create_json_metadata : Types.source -> Types.target list ->
> +                           Types.target_buses ->
> +                           Types.guestcaps ->
> +                           Types.inspect ->
> +                           Types.target_firmware ->
> +                           JSON.doc
> +(** [create_json_metadata source targets target_buses guestcaps
> +    inspect target_firmware] creates the JSON with the majority
> +    of the data that virt-v2v used for the conversion. *)
> diff --git a/v2v/output_json.ml b/v2v/output_json.ml
> new file mode 100644
> index 000000000..ca0bda978
> --- /dev/null
> +++ b/v2v/output_json.ml
> @@ -0,0 +1,116 @@
> +(* virt-v2v
> + * Copyright (C) 2019 Red Hat Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + *)
> +
> +open Printf
> +
> +open Std_utils
> +open Tools_utils
> +open Common_gettext.Gettext
> +
> +open Types
> +open Utils
> +
> +type json_options = {
> +  json_disks_pattern : string;
> +}
> +
> +let print_output_options () > +  printf (f_"Output options (-oo)
which can be used with -o json:
> +
> +  -oo json-disks-pattern=PATTERN   Pattern for the disks.
> +")
> +
> +let known_pattern_variables = ["DiskNo";
"DiskDeviceName"; "GuestName"]
> +
> +let parse_output_options options > +  let json_disks_pattern = ref None
in
> +
> +  List.iter (
> +    function
> +    | "json-disks-pattern", v ->
> +       if !json_disks_pattern <> None then
> +         error (f_"-o json: -oo json-disks-pattern set more than
once");
> +       let vars > +         try Var_expander.scan_variables v
> +         with Var_expander.Invalid_variable var ->
> +           error (f_"-o json: -oo json-disks-pattern: invalid
variable %%{%s}")
> +             var in
> +       List.iter (
> +         fun var ->
> +           if not (List.mem var known_pattern_variables) then
> +             error (f_"-o json: -oo json-disks-pattern: unhandled
variable %%{%s}")
> +               var
> +       ) vars;
> +       json_disks_pattern := Some v
> +    | k, _ ->
> +       error (f_"-o json: unknown output option ‘-oo %s’") k
> +  ) options;
> +
> +  let json_disks_pattern > +    Option.default
"%{GuestName}-%{DiskDeviceName}" !json_disks_pattern in
> +
> +  { json_disks_pattern }
> +
> +class output_json dir json_options = object
> +  inherit output
> +
> +  method as_options = sprintf "-o json -os %s" dir
> +
> +  method prepare_targets source overlays _ _ _ _ > +    List.mapi (
> +      fun i (_, ov) ->
> +        let outname > +          let vars_fn = function
> +            | "DiskNo" -> Some (string_of_int (i+1))
> +            | "DiskDeviceName" -> Some ov.ov_sd
> +            | "GuestName" -> Some source.s_name
> +            | _ -> assert false
> +          in
> +          Var_expander.replace_fn json_options.json_disks_pattern vars_fn
in
> +        let destname = dir // outname in
> +        mkdir_p (Filename.dirname destname) 0o755;
> +        TargetFile destname
> +    ) overlays
> +
> +  method supported_firmware = [ TargetBIOS; TargetUEFI ]
> +
> +  method create_metadata source targets
> +                         target_buses guestcaps inspect target_firmware
> +    let doc > +      Create_json.create_json_metadata source targets
target_buses
> +                                       guestcaps inspect target_firmware
in
> +    let doc_string = JSON.string_of_doc ~fmt:JSON.Indented doc in
> +
> +    if verbose () then (
> +      eprintf "resulting JSON:\n";
> +      output_string stderr doc_string;
> +      eprintf "\n\n%!";
> +    );
> +
> +    let name = source.s_name in
> +    let file = dir // name ^ ".json" in
> +
> +    with_open_out file (
> +      fun chan ->
> +        output_string chan doc_string;
> +        output_char chan '\n'
> +    )
> +end
> +
> +let output_json = new output_json
> +let () = Modules_list.register_output_module "json"
> diff --git a/v2v/output_json.mli b/v2v/output_json.mli
> new file mode 100644
> index 000000000..52f58f2d1
> --- /dev/null
> +++ b/v2v/output_json.mli
> @@ -0,0 +1,31 @@
> +(* virt-v2v
> + * Copyright (C) 2019 Red Hat Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + *)
> +
> +(** [-o json] target. *)
> +
> +type json_options
> +(** Miscellaneous extra command line parameters used by json. *)
> +
> +val print_output_options : unit -> unit
> +val parse_output_options : (string * string) list -> json_options
> +(** Print and parse json -oo options. *)
> +
> +val output_json : string -> json_options -> Types.output
> +(** [output_json directory json_options] creates and returns a new
> +    {!Types.output} object specialized for writing output to local
> +    files with JSON metadata. *)
> diff --git a/v2v/virt-v2v-output-local.pod b/v2v/virt-v2v-output-local.pod
> index 7427b1ed7..ec737e1ba 100644
> --- a/v2v/virt-v2v-output-local.pod
> +++ b/v2v/virt-v2v-output-local.pod
> @@ -11,6 +11,9 @@ or libvirt
>  
>   virt-v2v [-i* options] -o qemu -os DIRECTORY [--qemu-boot]
>  
> + virt-v2v [-i* options] -o json -os DIRECTORY
> +                        [-oo json-disks-pattern=PATTERN]
> +
>   virt-v2v [-i* options] -o null
>  
>  =head1 DESCRIPTION
> @@ -54,6 +57,13 @@ above, a shell script is created which contains the raw
qemu command
>  you would need to boot the guest.  However the shell script is not
>  run, I<unless> you also add the I<--qemu-boot> option.
>  
> +=item B<-o json -os> C<DIRECTORY>
> +
> +This converts the guest to files in C<DIRECTORY>.  The metadata
> +produced is a JSON file containing the majority of the data virt-v2v
> +gathers during the conversion.
> +See L</OUTPUT TO JSON> below.
> +
>  =item B<-o null>
>  
>  The guest is converted, but the final result is thrown away and no
> @@ -140,6 +150,46 @@ Define the final guest in libvirt:
>  
>  =back
>  
> +=head1 OUTPUT TO JSON
> +
> +The I<-o json> option produces the following files by default:
> +
> + NAME.json                     JSON metadata.
> + NAME-sda, NAME-sdb, etc.      Guest disk(s).
> +
> +where C<NAME> is the guest name.
> +
> +It is possible to change the pattern of the disks using the
> +I<-oo json-disks-pattern=...> option: it allows parameters in form
of
> +C<%{...}> variables, for example:
> +
> + -oo json-disks-pattern=disk%{DiskNo}.img
> +
> +Recognized variables are:
> +
> +=over 4
> +
> +=item C<%{DiskNo}>
> +
> +The index of the disk, starting from 1.
> +
> +=item C<%{DiskDeviceName}>
> +
> +The destination device of the disk, e.g. C<sda>, C<sdb>, etc.
> +
> +=item C<%{GuestName}>
> +
> +The name of the guest.
> +
> +=back
> +
> +Using a pattern it is possible use subdirectories for the disks,
> +even with names depending on variables; for example:
> +
> + -oo json-disks-pattern=%{GuestName}-%{DiskNo}/disk.img
> +
> +The default pattern is C<%{GuestName}-%{DiskDeviceName}>.
> +
>  =head1 SEE ALSO
>  
>  L<virt-v2v(1)>.
> diff --git a/v2v/virt-v2v.pod b/v2v/virt-v2v.pod
> index cf9464834..9a555c3be 100644
> --- a/v2v/virt-v2v.pod
> +++ b/v2v/virt-v2v.pod
> @@ -425,6 +425,17 @@ instead.
>  Set the output method to OpenStack Glance.  In this mode the converted
>  guest is uploaded to Glance.  See L<virt-v2v-output-openstack(1)>.
>  
> +=item B<-o> B<json>
> +
> +Set the output method to I<json>.
> +
> +In this mode, the converted guest is written to a local directory
> +specified by I<-os /dir> (the directory must exist), with a JSON
file
> +containing the majority of the metadata that virt-v2v gathered during
> +the conversion.
> +
> +See L<virt-v2v-output-local(1)>.
> +
>  =item B<-o> B<libvirt>
>  
>  Set the output method to I<libvirt>.  This is the default.
> @@ -696,8 +707,8 @@ The location of the storage for the converted guest.
>  For I<-o libvirt>, this is a libvirt directory pool
>  (see S<C<virsh pool-list>>) or pool UUID.
>  
> -For I<-o local> and I<-o qemu>, this is a directory name.  The
> -directory must exist.
> +For I<-o json>, I<-o local> and I<-o qemu>, this is a
directory name.
> +The directory must exist.
>  
>  For I<-o rhv-upload>, this is the name of the destination Storage
>  Domain.
> -- 
> 2.20.1
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs
-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
Fedora Windows cross-compiler. Compile Windows programs, test, and
build Windows installers. Over 100 libraries supported.
http://fedoraproject.org/wiki/MinGW