Libguestfs - Jan 2015 - [PATCH] customize: add --commands-from-file

Pass to --commands-from-file the name of a file containing customization
commands in each line, as if they were specified as command line
arguments.

This eases the reuse of commands among different
builder/customize/sysprep invocations.
---
 builder/cmdline.ml         |  3 +-
 customize/customize_run.ml |  5 +++
 generator/customize.ml     | 98 ++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 101 insertions(+), 5 deletions(-)

diff --git a/builder/cmdline.ml b/builder/cmdline.ml
index bb7b1d0..1c6ab98 100644
--- a/builder/cmdline.ml
+++ b/builder/cmdline.ml
@@ -308,7 +308,8 @@ read the man page virt-builder(1).
           | `Delete _ | `Edit _ | `FirstbootCommand _ | `FirstbootPackages _
           | `FirstbootScript _ | `Hostname _ | `Link _ | `Mkdir _
           | `Password _ | `RootPassword _ | `Scrub _ | `SSHInject _
-          | `Timezone _ | `Upload _ | `Write _ | `Chmod _ -> false
+          | `Timezone _ | `Upload _ | `Write _ | `Chmod _
+          | `CommandsFromFile _ -> false
         ) ops.ops in
         if requires_execute_on_guest then
           error (f_"sorry, cannot run commands on a guest with a different
architecture");
diff --git a/customize/customize_run.ml b/customize/customize_run.ml
index 19b7a7d..fed905b 100644
--- a/customize/customize_run.ml
+++ b/customize/customize_run.ml
@@ -170,6 +170,11 @@ exec >>%s 2>&1
       msg (f_"Running: %s") cmd;
       do_run ~display:cmd cmd
 
+    | `CommandsFromFile _ ->
+      (* Nothing to do, the files with customize commands are already
+       * read when their arguments are met. *)
+      ()
+
     | `Delete path ->
       msg (f_"Deleting: %s") path;
       g#rm_rf path
diff --git a/generator/customize.ml b/generator/customize.ml
index 82ecb79..d5e72ac 100644
--- a/generator/customize.ml
+++ b/generator/customize.ml
@@ -43,6 +43,7 @@ and op_type  | PasswordSelector of string            (*
password selector *)
 | UserPasswordSelector of string        (* user:selector *)
 | SSHKeySelector of string              (* user:selector *)
+| StringFn of (string * string)         (* string, function name *)
 
 let ops = [
   { op_name = "chmod";
@@ -56,6 +57,29 @@ I<Note>: C<PERMISSIONS> by default would be
decimal, unless you prefix
 it with C<0> to get octal, ie. use C<0700> not C<700>.";
   };
 
+  { op_name = "commands-from-file";
+    op_type = StringFn ("FILENAME",
"customize_read_from_file");
+    op_discrim = "`CommandsFromFile";
+    op_shortdesc = "Read customize commands from file";
+    op_pod_longdesc = "\
+Read the customize commands from a file, one (and its arguments)
+each line.
+
+Each line contains a single customization command and its arguments,
+for example:
+
+ delete /some/file
+ install some-package
+ password some-user:password:its-new-password
+
+Empty lines are ignored, and lines starting with C<#> are comments
+and are ignored as well.
+
+The commands are handled in the same order as they are in the file,
+as if they were specified as I<--delete /some/file> on the command
+line.";
+  };
+
   { op_name = "delete";
     op_type = String "PATH";
     op_discrim = "`Delete";
@@ -474,7 +498,7 @@ let rec argspec ()      | target :: lns -> target, lns
   in
 
-  let argspec = [
+  let rec argspec = [
 ";
 
   List.iter (
@@ -569,6 +593,18 @@ let rec argspec ()        pr "      s_\"%s\"
^ \" \" ^ s_\"%s\"\n" v shortdesc;
       pr "    ),\n";
       pr "    Some %S, %S;\n" v longdesc
+    | { op_type = StringFn (v, fn); op_name = name; op_discrim = discrim;
+        op_shortdesc = shortdesc; op_pod_longdesc = longdesc } ->
+      pr "    (\n";
+      pr "      \"--%s\",\n" name;
+      pr "      Arg.String (\n";
+      pr "        fun s ->\n";
+      pr "          %s s;\n" fn;
+      pr "          ops := %s s :: !ops\n" discrim;
+      pr "      ),\n";
+      pr "      s_\"%s\" ^ \" \" ^
s_\"%s\"\n" v shortdesc;
+      pr "    ),\n";
+      pr "    Some %S, %S;\n" v longdesc
   ) ops;
 
   List.iter (
@@ -598,7 +634,57 @@ let rec argspec ()        pr "    Some %S, %S;\n"
v longdesc
   ) flags;
 
-  pr "  ] in
+  pr "  ]
+  and customize_read_from_file filename +    let forbidden_commands = [
+";
+
+  List.iter (
+    function
+    | { op_type = StringFn (_, _); op_name = name; } ->
+      pr "      \"%s\";\n" name
+    | { op_type = Unit; }
+    | { op_type = String _; }
+    | { op_type = StringPair _; }
+    | { op_type = StringList _; }
+    | { op_type = TargetLinks _; }
+    | { op_type = PasswordSelector _; }
+    | { op_type = UserPasswordSelector _; }
+    | { op_type = SSHKeySelector _; } -> ()
+  ) ops;
+
+pr "    ] in
+    let lines = read_whole_file filename in
+    let lines = string_nsplit \"\\n\" lines in
+    let lines = List.filter (
+      fun line ->
+        String.length line > 0 && line.[0] <> '#'
+    ) lines in
+    let cmds = List.map (fun line -> string_split \" \" line)
lines in
+    (* Check for commands not allowed in files containing commands. *)
+    List.iter (
+      fun (cmd, _) ->
+        if List.mem cmd forbidden_commands then
+          error (f_\"command '%%s' cannot be used in command
files, see the man page\")
+            cmd
+    ) cmds;
+    List.iter (
+      fun (cmd, arg) ->
+        try
+          let ((_, spec, _), _, _) = List.find (
+            fun ((key, _, _), _, _) ->
+              key = \"--\" ^ cmd
+          ) argspec in
+          (match spec with
+          | Arg.Unit fn -> fn ()
+          | Arg.String fn -> fn arg
+          | _ -> error \"INTERNAL error: spec not handled for
%%s\" cmd
+          )
+        with Not_found ->
+          error (f_\"command '%%s' not valid, see the man
page\")
+            cmd
+    ) cmds
+  in
 
   argspec, get_ops
 "
@@ -640,6 +726,8 @@ type ops = {
         op_name = name } ->
       pr "  | %s of string * Ssh_key.ssh_key_selector\n      (* --%s %s
*)\n"
         discrim name v
+    | { op_type = StringFn (v, _); op_discrim = discrim; op_name = name } ->
+      pr "  | %s of string\n      (* --%s %s *)\n" discrim name v
   ) ops;
   pr "]\n";
 
@@ -665,7 +753,8 @@ let generate_customize_synopsis_pod ()        | { op_type =
Unit; op_name = n } ->
         n, sprintf "[--%s]" n
       | { op_type = String v | StringPair v | StringList v | TargetLinks v
-            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v;
+            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v
+            | StringFn (v, _);
           op_name = n } ->
         n, sprintf "[--%s %s]" n v
     ) ops @
@@ -705,7 +794,8 @@ let generate_customize_options_pod ()        | { op_type =
Unit; op_name = n; op_pod_longdesc = ld } ->
         n, sprintf "B<--%s>" n, ld
       | { op_type = String v | StringPair v | StringList v | TargetLinks v
-            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v;
+            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v
+            | StringFn (v, _);
           op_name = n; op_pod_longdesc = ld } ->
         n, sprintf "B<--%s> %s" n v, ld
     ) ops @
-- 
1.9.3

On Wed, Jan 21, 2015 at 04:32:39PM +0100, Pino Toscano
wrote:
> Pass to --commands-from-file the name of a file containing customization
> commands in each line, as if they were specified as command line
> arguments.
I like the idea, but the file format needs to be more structured.
It's very common for --edit and --run-command to be split across
multiple lines (example: [1]).

What other options are there for this file?  Something that can deal
with multi-line input but without excessive escaping.

Rich.

[1]
https://rwmj.wordpress.com/2014/01/13/tip-set-the-keyboard-layout-in-virt-builder/

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-builder quickly builds VMs from scratch
http://libguestfs.org/virt-builder.1.html

Pass to --commands-from-file the name of a file containing customization
commands in each line, as if they were specified as command line
arguments.

This eases the reuse of commands among different
builder/customize/sysprep invocations.
---
 builder/cmdline.ml         |   3 +-
 customize/customize_run.ml |   5 ++
 generator/customize.ml     | 131 +++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 134 insertions(+), 5 deletions(-)

diff --git a/builder/cmdline.ml b/builder/cmdline.ml
index bb7b1d0..1c6ab98 100644
--- a/builder/cmdline.ml
+++ b/builder/cmdline.ml
@@ -308,7 +308,8 @@ read the man page virt-builder(1).
           | `Delete _ | `Edit _ | `FirstbootCommand _ | `FirstbootPackages _
           | `FirstbootScript _ | `Hostname _ | `Link _ | `Mkdir _
           | `Password _ | `RootPassword _ | `Scrub _ | `SSHInject _
-          | `Timezone _ | `Upload _ | `Write _ | `Chmod _ -> false
+          | `Timezone _ | `Upload _ | `Write _ | `Chmod _
+          | `CommandsFromFile _ -> false
         ) ops.ops in
         if requires_execute_on_guest then
           error (f_"sorry, cannot run commands on a guest with a different
architecture");
diff --git a/customize/customize_run.ml b/customize/customize_run.ml
index 19b7a7d..fed905b 100644
--- a/customize/customize_run.ml
+++ b/customize/customize_run.ml
@@ -170,6 +170,11 @@ exec >>%s 2>&1
       msg (f_"Running: %s") cmd;
       do_run ~display:cmd cmd
 
+    | `CommandsFromFile _ ->
+      (* Nothing to do, the files with customize commands are already
+       * read when their arguments are met. *)
+      ()
+
     | `Delete path ->
       msg (f_"Deleting: %s") path;
       g#rm_rf path
diff --git a/generator/customize.ml b/generator/customize.ml
index 82ecb79..d9865f5 100644
--- a/generator/customize.ml
+++ b/generator/customize.ml
@@ -43,6 +43,7 @@ and op_type  | PasswordSelector of string            (*
password selector *)
 | UserPasswordSelector of string        (* user:selector *)
 | SSHKeySelector of string              (* user:selector *)
+| StringFn of (string * string)         (* string, function name *)
 
 let ops = [
   { op_name = "chmod";
@@ -56,6 +57,34 @@ I<Note>: C<PERMISSIONS> by default would be
decimal, unless you prefix
 it with C<0> to get octal, ie. use C<0700> not C<700>.";
   };
 
+  { op_name = "commands-from-file";
+    op_type = StringFn ("FILENAME",
"customize_read_from_file");
+    op_discrim = "`CommandsFromFile";
+    op_shortdesc = "Read customize commands from file";
+    op_pod_longdesc = "\
+Read the customize commands from a file, one (and its arguments)
+each line.
+
+Each line contains a single customization command and its arguments,
+for example:
+
+ delete /some/file
+ install some-package
+ password some-user:password:its-new-password
+
+Empty lines are ignored, and lines starting with C<#> are comments
+and are ignored as well.  Furthermore, arguments can be spread across
+multiple lines, by adding a C<\\> (continuation character) at the of
+a line, for example
+
+ edit /some/file:\\
+   s/^OPT=.*/OPT=ok/
+
+The commands are handled in the same order as they are in the file,
+as if they were specified as I<--delete /some/file> on the command
+line.";
+  };
+
   { op_name = "delete";
     op_type = String "PATH";
     op_discrim = "`Delete";
@@ -474,7 +503,7 @@ let rec argspec ()      | target :: lns -> target, lns
   in
 
-  let argspec = [
+  let rec argspec = [
 ";
 
   List.iter (
@@ -569,6 +598,18 @@ let rec argspec ()        pr "      s_\"%s\"
^ \" \" ^ s_\"%s\"\n" v shortdesc;
       pr "    ),\n";
       pr "    Some %S, %S;\n" v longdesc
+    | { op_type = StringFn (v, fn); op_name = name; op_discrim = discrim;
+        op_shortdesc = shortdesc; op_pod_longdesc = longdesc } ->
+      pr "    (\n";
+      pr "      \"--%s\",\n" name;
+      pr "      Arg.String (\n";
+      pr "        fun s ->\n";
+      pr "          %s s;\n" fn;
+      pr "          ops := %s s :: !ops\n" discrim;
+      pr "      ),\n";
+      pr "      s_\"%s\" ^ \" \" ^
s_\"%s\"\n" v shortdesc;
+      pr "    ),\n";
+      pr "    Some %S, %S;\n" v longdesc
   ) ops;
 
   List.iter (
@@ -598,7 +639,85 @@ let rec argspec ()        pr "    Some %S, %S;\n"
v longdesc
   ) flags;
 
-  pr "  ] in
+  pr "  ]
+  and customize_read_from_file filename +    let rec string_lines_split str +  
let buf = Buffer.create 16 in
+      let len = String.length str in
+      let rec loop start len +        try
+          let i = String.index_from str start '\\n' in
+          if i > 0 && str.[i-1] = '\\\\' then (
+            Buffer.add_substring buf str start (i-start-1);
+            Buffer.add_char buf '\\n';
+            loop (i+1) len
+          ) else (
+            Buffer.add_substring buf str start (i-start);
+            i+1
+          )
+        with Not_found ->
+          if len > 0 && str.[len-1] = '\\\\' then (
+            Buffer.add_substring buf str start (len-start-1);
+            Buffer.add_char buf '\\n'
+          ) else
+            Buffer.add_substring buf str start (len-start);
+          len+1
+      in
+      let endi = loop 0 len in
+      let line = Buffer.contents buf in
+      if endi > len then
+        [line]
+      else
+        line :: string_lines_split (String.sub str endi (len-endi)) in
+    let forbidden_commands = [
+";
+
+  List.iter (
+    function
+    | { op_type = StringFn (_, _); op_name = name; } ->
+      pr "      \"%s\";\n" name
+    | { op_type = Unit; }
+    | { op_type = String _; }
+    | { op_type = StringPair _; }
+    | { op_type = StringList _; }
+    | { op_type = TargetLinks _; }
+    | { op_type = PasswordSelector _; }
+    | { op_type = UserPasswordSelector _; }
+    | { op_type = SSHKeySelector _; } -> ()
+  ) ops;
+
+pr "    ] in
+    let lines = read_whole_file filename in
+    let lines = string_lines_split lines in
+    let lines = List.filter (
+      fun line ->
+        String.length line > 0 && line.[0] <> '#'
+    ) lines in
+    let cmds = List.map (fun line -> string_split \" \" line)
lines in
+    (* Check for commands not allowed in files containing commands. *)
+    List.iter (
+      fun (cmd, _) ->
+        if List.mem cmd forbidden_commands then
+          error (f_\"command '%%s' cannot be used in command
files, see the man page\")
+            cmd
+    ) cmds;
+    List.iter (
+      fun (cmd, arg) ->
+        try
+          let ((_, spec, _), _, _) = List.find (
+            fun ((key, _, _), _, _) ->
+              key = \"--\" ^ cmd
+          ) argspec in
+          (match spec with
+          | Arg.Unit fn -> fn ()
+          | Arg.String fn -> fn arg
+          | _ -> error \"INTERNAL error: spec not handled for
%%s\" cmd
+          )
+        with Not_found ->
+          error (f_\"command '%%s' not valid, see the man
page\")
+            cmd
+    ) cmds
+  in
 
   argspec, get_ops
 "
@@ -640,6 +759,8 @@ type ops = {
         op_name = name } ->
       pr "  | %s of string * Ssh_key.ssh_key_selector\n      (* --%s %s
*)\n"
         discrim name v
+    | { op_type = StringFn (v, _); op_discrim = discrim; op_name = name } ->
+      pr "  | %s of string\n      (* --%s %s *)\n" discrim name v
   ) ops;
   pr "]\n";
 
@@ -665,7 +786,8 @@ let generate_customize_synopsis_pod ()        | { op_type =
Unit; op_name = n } ->
         n, sprintf "[--%s]" n
       | { op_type = String v | StringPair v | StringList v | TargetLinks v
-            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v;
+            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v
+            | StringFn (v, _);
           op_name = n } ->
         n, sprintf "[--%s %s]" n v
     ) ops @
@@ -705,7 +827,8 @@ let generate_customize_options_pod ()        | { op_type =
Unit; op_name = n; op_pod_longdesc = ld } ->
         n, sprintf "B<--%s>" n, ld
       | { op_type = String v | StringPair v | StringList v | TargetLinks v
-            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v;
+            | PasswordSelector v | UserPasswordSelector v | SSHKeySelector v
+            | StringFn (v, _);
           op_name = n; op_pod_longdesc = ld } ->
         n, sprintf "B<--%s> %s" n v, ld
     ) ops @
-- 
1.9.3