Libguestfs - Feb 2017 - Re: [PATCH 06/10] copy-out: new 'excludes' optional argument

Hi,

this patch series does changes mostly in virt-dib, few bug fixes and
a couple of new features (mostly implemented upstream already).
In addition, one new API is added, and a new optional argument for an
existing API is added (the latter is not needed, but could be useful
anyway).

Thanks,

Pino Toscano (10):
  dib: fix listing envvars in fake-sudo
  dib: source dib "die" script in some phases
  dib: change hooks path when running extra-data.d scripts
  dib: re-read list of scripts when running in-chroot hooks
  dib: cleanup logs at end of build
  copy-out: new 'excludes' optional argument
  dib: add appliance check hook in Output_format
  daemon: move make_exclude_from_file as common helper
  New API: mksquashfs
  dib: add squashfs output format

 appliance/packagelist.in      |   2 +
 daemon/Makefile.am            |   1 +
 daemon/daemon.h               |   2 +
 daemon/guestfsd.c             |  62 +++++++++++++++++
 daemon/squashfs.c             | 150 ++++++++++++++++++++++++++++++++++++++++++
 daemon/tar.c                  |  58 +---------------
 dib/Makefile.am               |   1 +
 dib/dib.ml                    |  49 +++++++++++---
 dib/elements.ml               |  15 +++++
 dib/output_format.ml          |  15 +++++
 dib/output_format.mli         |   9 +++
 dib/output_format_squashfs.ml |  39 +++++++++++
 dib/utils.ml                  |   1 +
 dib/virt-dib.pod              |   6 ++
 generator/actions.ml          |  42 +++++++++++-
 gobject/Makefile.inc          |   4 ++
 lib/MAX_PROC_NR               |   2 +-
 lib/copy-in-out.c             |  13 +++-
 18 files changed, 398 insertions(+), 73 deletions(-)
 create mode 100644 daemon/squashfs.c
 create mode 100644 dib/output_format_squashfs.ml

-- 
2.9.3

Query awk for the list of environment variables, instead of trying to
extract the list from the output of `env`: the old approach breaks when
any of the environment variable contains more than one line.
---
 dib/dib.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dib/dib.ml b/dib/dib.ml
index 71b1f7f..54ea2ae 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -289,7 +289,7 @@ if [ -n \"$user\" ]; then
 fi
 
 if [ -z \"$preserve_env\" ]; then
-  for envvar in `env | grep '^\\w' | cut -d= -f1`; do
+  for envvar in `awk 'BEGIN{for (i in ENVIRON) {print i}}'`; do
     case \"$envvar\" in
       PATH | USER | USERNAME | HOSTNAME | TERM | LANG | HOME | SHELL | LOGNAME
) ;;
       BASH_FUNC_* ) unset -f $envvar ;;
-- 
2.9.3

Source the "die" script, part of the diskimage-builder library, when
running hooks that run outside the guest chroot.  This is what
disk-image-create does, and scripts expect to use the "die" function
without sourcing the "die" script containing it.
---
 dib/dib.ml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dib/dib.ml b/dib/dib.ml
index 54ea2ae..6a0145d 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -124,6 +124,8 @@ if [ -d $ENVIRONMENT_D_DIR ] ; then
     done
 fi
 
+source $_LIB/die
+
 $target_dir/$script
 "
     (if debug >= 1 then "set -x\n" else "")
@@ -206,6 +208,7 @@ export TMP_IMAGE_DIR=$mysysroot/tmp/aux
 
 if [ -n \"$mysysroot\" ]; then
   export PATH=$mysysroot/tmp/aux/fake-bin:$PATH
+  source $_LIB/die
 else
   export
PATH=\"$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"
 fi
-- 
2.9.3

It looks like scripts (in particular in the extra-data.d phase) may
in-flight add new scripts to the existing ones.  As first step to
handle this situation, change the path of hooks passed for extra-data.d
scripts: instead of the local temporary directory, use the in-guest
location that is visible as result of the local fuse-based mount of the
guest.
---
 dib/dib.ml | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/dib/dib.ml b/dib/dib.ml
index 6a0145d..9884a7a 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -73,7 +73,7 @@ let envvars_string l  
 let prepare_external ~envvars ~dib_args ~dib_vars ~out_name ~root_label
   ~rootfs_uuid ~image_cache ~arch ~network ~debug ~fs_type ~checksum
-  destdir libdir hooksdir fakebindir all_elements element_paths +  destdir
libdir fakebindir all_elements element_paths    let network_string = if network
then "" else "1" in
   let checksum_string = if checksum then "1" else "" in
 
@@ -83,6 +83,8 @@ set -e
 %s
 mount_dir=$1
 shift
+hooks_dir=$1
+shift
 target_dir=$1
 shift
 script=$1
@@ -102,7 +104,7 @@ export DIB_IMAGE_ROOT_FS_UUID=%s
 export DIB_IMAGE_CACHE=\"%s\"
 export _LIB=%s
 export ARCH=%s
-export TMP_HOOKS_PATH=%s
+export TMP_HOOKS_PATH=\"$hooks_dir\"
 export DIB_ARGS=\"%s\"
 export IMAGE_ELEMENT=\"%s\"
 export ELEMENTS_PATH=\"%s\"
@@ -138,7 +140,6 @@ $target_dir/$script
     image_cache
     (quote libdir)
     arch
-    (quote hooksdir)
     dib_args
     (String.concat " " (StringSet.elements all_elements))
     (String.concat ":" element_paths)
@@ -407,10 +408,14 @@ let run_parts ~debug ~sysroot ~blockdev ~log_file ?(new_wd
= "")
   flush_all ();
   Buffer.contents outbuf
 
-let run_parts_host ~debug (g : Guestfs.guestfs) hooks_dir hook_name
base_mount_dir scripts run_script -  let hook_dir = hooks_dir // hook_name in
+let run_parts_host ~debug (g : Guestfs.guestfs) hook_name base_mount_dir
scripts run_script    let scripts = List.sort digit_prefix_compare scripts in
   let mount_dir = base_mount_dir // hook_name in
+  (* Point to the in-guest hooks, so that changes there can affect
+   * other phases.
+   *)
+  let hooks_dir = mount_dir // "tmp" // "aux" //
"hooks" in
+  let hook_dir = hooks_dir // hook_name in
   do_mkdir mount_dir;
 
   let rec fork_and_run () @@ -428,7 +433,7 @@ let run_parts_host ~debug (g :
Guestfs.guestfs) hooks_dir hook_name base_mount_d
     let rec loop = function
       | x :: xs ->
         message (f_"Running: %s/%s") hook_name x;
-        let cmd = [ run_script; mount_dir; hook_dir; x ] in
+        let cmd = [ run_script; mount_dir; hooks_dir; hook_dir; x ] in
         let retcode = ref 0 in
         let run ()            retcode := run_command cmd in
@@ -623,7 +628,7 @@ let main ()                     ~network:cmdline.network
~debug
                    ~fs_type:cmdline.fs_type
                    ~checksum:cmdline.checksum
-                   tmpdir cmdline.basepath hookstmpdir
+                   tmpdir cmdline.basepath
                    (auxtmpdir // "fake-bin")
                    all_elements cmdline.element_paths;
 
@@ -779,7 +784,7 @@ let main ()        if debug >= 1 then (
         printf "Running hooks for %s...\n%!" hook;
       );
-      run_parts_host ~debug g hookstmpdir hook tmpdir scripts
+      run_parts_host ~debug g hook tmpdir scripts
         (tmpdir // "run-part-extra.sh")
     with Not_found -> () in
 
-- 
2.9.3

Scripts (especially root.d and extra-data.d) may add new scripts to the
existing set during their execution: in this case, re-read the list of
available scripts, so we do not miss any new addition.
---
 dib/dib.ml      | 14 +++++++++++++-
 dib/elements.ml | 15 +++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/dib/dib.ml b/dib/dib.ml
index 9884a7a..ff2298d 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -634,7 +634,19 @@ let main ()  
   let run_hook ~blockdev ~sysroot ?(new_wd = "") (g :
Guestfs.guestfs) hook      try
-      let scripts = Hashtbl.find final_hooks hook in
+      let scripts +        (* Sadly, scripts (especially in root.d and
extra-data.d)
+         * can add (by copying or symlinking) new scripts for other
+         * phases, which would be ignored if we were using the lists
+         * collected after composing the tree of hooks.
+         * As result, when running in-chroot hooks, re-read the list
+         * of scripts actually available for each hook.
+         *)
+        match hook with
+        | "pre-install.d" | "install.d" |
"post-install.d" | "finalise.d" ->
+          load_scripts g ("/tmp/aux/hooks/" ^ hook)
+        | _ ->
+          Hashtbl.find final_hooks hook in
       if debug >= 1 then (
         printf "Running hooks for %s...\n%!" hook;
       );
diff --git a/dib/elements.ml b/dib/elements.ml
index c6ca0ea..4c2875a 100644
--- a/dib/elements.ml
+++ b/dib/elements.ml
@@ -80,6 +80,21 @@ let load_hooks ~debug path    ) entries;
   hooks
 
+let load_scripts (g : Guestfs.guestfs) path +  let listing = Array.to_list
(g#readdir path) in
+  let scripts = List.filter (
+    function
+    | { Guestfs.ftyp = ('r'|'l'|'u'|'?') }
-> true
+    | _ -> false
+    ) listing in
+  let scripts = List.filter (fun x -> valid_script_name x.Guestfs.name)
scripts in
+  filter_map (
+     fun x ->
+       let { Guestfs.st_mode = mode } = g#statns (path ^ "/" ^
x.Guestfs.name) in
+       if mode &^ 0o111_L > 0_L then Some x.Guestfs.name
+       else None
+   ) scripts
+
 let load_elements ~debug paths    let loaded_elements = Hashtbl.create 13 in
   let paths = List.filter is_directory paths in
-- 
2.9.3

Recently, diskimage-builder moved the log cleanup from the 'base'
element to disk-image-create proper; the cleanup done is:
- truncate any file in /var/log
- remove *.log files in /root

This was implemented in diskimage-builder upstream as
commit 022d93ee822e71245af52c4cf8f8a8e82f599af3.
---
 dib/dib.ml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/dib/dib.ml b/dib/dib.ml
index ff2298d..b578750 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -895,6 +895,13 @@ let main ()    flush_all ();
   g#mount blockdev "/";
   Array.iter (fun x -> g#rm_rf ("/tmp/" ^ x)) (g#ls
"/tmp");
+  (* Truncate /var/log files in preparation for first boot. *)
+  truncate_recursive g "/var/log";
+  let non_log fn +    not (String.is_suffix fn ".log")
+  in
+  (* Remove root logs. *)
+  rm_rf_only_files g ~filter:non_log "/root";
 
   flush_all ();
 
-- 
2.9.3

Add a new 'excludes' optional argument to copy-out, passing it straight
to tar-out: this way it is possible to exclude files when extracting a
directory from the guest.
---
 generator/actions.ml | 16 ++++++++++++++--
 gobject/Makefile.inc |  2 ++
 lib/copy-in-out.c    | 13 ++++++++++---
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/generator/actions.ml b/generator/actions.ml
index 990eacb..fd6cc9f 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -3437,8 +3437,9 @@ Wildcards cannot be used." };
 
   { defaults with
     name = "copy_out"; added = (1, 29, 24);
-    style = RErr, [Pathname "remotepath"; String
"localdir"], [];
+    style = RErr, [Pathname "remotepath"; String
"localdir"], [OStringList "excludes"];
     visibility = VPublicNoFish;
+    once_had_no_optargs = true;
     shortdesc = "copy remote files or directories out of an image";
     longdesc = "\
 C<guestfs_copy_out> copies remote files or directories recursively
@@ -3449,7 +3450,18 @@ To download to the current directory, use C<.> as
in:
 
  C<guestfs_copy_out> /home .
 
-Wildcards cannot be used." };
+Wildcards cannot be used.
+
+The other optional arguments are:
+
+=over 4
+
+=item C<excludes>
+
+A list of wildcards.  Files are excluded if they match any of the
+wildcards.  Note they are used only if C<remotepath> is a directory.
+
+=back" };
 
   { defaults with
     name = "set_identifier"; added = (1, 31, 14);
diff --git a/gobject/Makefile.inc b/gobject/Makefile.inc
index 55a137e..76482f7 100644
--- a/gobject/Makefile.inc
+++ b/gobject/Makefile.inc
@@ -67,6 +67,7 @@ guestfs_gobject_headers= \
   include/guestfs-gobject/optargs-copy_device_to_file.h \
   include/guestfs-gobject/optargs-copy_file_to_device.h \
   include/guestfs-gobject/optargs-copy_file_to_file.h \
+  include/guestfs-gobject/optargs-copy_out.h \
   include/guestfs-gobject/optargs-cpio_out.h \
   include/guestfs-gobject/optargs-disk_create.h \
   include/guestfs-gobject/optargs-download_blocks.h \
@@ -158,6 +159,7 @@ guestfs_gobject_sources= \
   src/optargs-copy_device_to_file.c \
   src/optargs-copy_file_to_device.c \
   src/optargs-copy_file_to_file.c \
+  src/optargs-copy_out.c \
   src/optargs-cpio_out.c \
   src/optargs-disk_create.c \
   src/optargs-download_blocks.c \
diff --git a/lib/copy-in-out.c b/lib/copy-in-out.c
index a4e39f8..4691ead 100644
--- a/lib/copy-in-out.c
+++ b/lib/copy-in-out.c
@@ -131,8 +131,9 @@ child_setup (guestfs_h *g, void *data)
 }
 
 int
-guestfs_impl_copy_out (guestfs_h *g,
-                       const char *remotepath, const char *localdir)
+guestfs_impl_copy_out_opts (guestfs_h *g,
+                            const char *remotepath, const char *localdir,
+                            const struct guestfs_copy_out_opts_argv *optargs)
 {
   struct stat statbuf;
   int r;
@@ -170,6 +171,7 @@ guestfs_impl_copy_out (guestfs_h *g,
     struct copy_out_child_data data;
     char fdbuf[64];
     int fd;
+    struct guestfs_tar_out_opts_argv tar_optargs = { .bitmask = 0 };
 
     r = guestfs_is_dir (g, remotepath);
     if (r == -1)
@@ -210,7 +212,12 @@ guestfs_impl_copy_out (guestfs_h *g,
 
     snprintf (fdbuf, sizeof fdbuf, "/dev/fd/%d", fd);
 
-    r = guestfs_tar_out (g, remotepath, fdbuf);
+    if (optargs->bitmask & GUESTFS_COPY_OUT_OPTS_EXCLUDES_BITMASK) {
+      tar_optargs.excludes = optargs->excludes;
+      tar_optargs.bitmask |= GUESTFS_TAR_OUT_OPTS_EXCLUDES_BITMASK;
+    }
+
+    r = guestfs_tar_out_opts_argv (g, remotepath, fdbuf, &tar_optargs);
 
     if (close (fd) == -1) {
       perrorf (g, "close (tar-output subprocess)");
-- 
2.9.3

Add a new hook for Output_format's to check for prerequisites on the
appliance (which can be done only when the appliance is run).
---
 dib/dib.ml            |  2 ++
 dib/output_format.ml  | 15 +++++++++++++++
 dib/output_format.mli |  9 +++++++++
 3 files changed, 26 insertions(+)

diff --git a/dib/dib.ml b/dib/dib.ml
index b578750..df83ba1 100644
--- a/dib/dib.ml
+++ b/dib/dib.ml
@@ -703,6 +703,8 @@ let main ()  
     g#launch ();
 
+    Output_format.check_formats_appliance_prerequisites cmdline.formats g;
+
     (* Prepare the /aux partition. *)
     g#mkfs "ext2" "/dev/sdb";
     g#mount "/dev/sdb" "/";
diff --git a/dib/output_format.ml b/dib/output_format.ml
index f311b8d..851cefc 100644
--- a/dib/output_format.ml
+++ b/dib/output_format.ml
@@ -27,6 +27,7 @@ type format = {
   extra_args : extra_arg list;
   output_to_file : bool;
   check_prerequisites : (unit -> unit) option;
+  check_appliance_prerequisites : (Guestfs.guestfs -> unit) option;
   run_on_filesystem : (Guestfs.guestfs -> string -> string -> unit)
option;
   run_on_file : (string -> (string * string) -> string -> unit)
option;
 }
@@ -39,6 +40,7 @@ let defaults = {
   extra_args = [];
   output_to_file = true;
   check_prerequisites = None;
+  check_appliance_prerequisites = None;
   run_on_filesystem = None;
   run_on_file = None;
 }
@@ -129,6 +131,19 @@ let check_formats_prerequisites ~formats      | {
check_prerequisites = None } -> ()
   ) formats
 
+let check_formats_appliance_prerequisites ~formats g +  assert !baked;
+
+  (* Run the formats in alphabetical, rather than random order. *)
+  let formats = List.sort compare_formats (FormatSet.elements formats) in
+
+  List.iter (
+    function
+    | { check_appliance_prerequisites = Some fn } ->
+      fn g
+    | { check_appliance_prerequisites = None } -> ()
+  ) formats
+
 let run_formats_on_filesystem ~formats g image_name tmpdir    assert !baked;
 
diff --git a/dib/output_format.mli b/dib/output_format.mli
index e935f6f..76683ad 100644
--- a/dib/output_format.mli
+++ b/dib/output_format.mli
@@ -44,6 +44,11 @@ type format = {
       to check whether the requirements for this format (available
       tools, values for command line arguments, etc) are fulfilled. *)
 
+  check_appliance_prerequisites : (Guestfs.guestfs -> unit) option;
+  (** The function which is called after the appliance start to check
+      whether the requirements in the appliance for this format
+      (available features, filesystems, etc) are fulfilled. *)
+
   run_on_filesystem : (Guestfs.guestfs -> string -> string -> unit)
option;
   (** The function which is called to perform the export while the
       guest is mounted.
@@ -108,6 +113,10 @@ val set_cardinal : set -> int
 val check_formats_prerequisites : formats:set -> unit
 (** Check the prerequisites in all the formats listed in the [formats] set. *)
 
+val check_formats_appliance_prerequisites : formats:set -> Guestfs.guestfs
-> unit
+(** Check the appliance prerequisites in all the formats listed in the
+    [formats] set. *)
+
 val run_formats_on_filesystem : formats:set -> Guestfs.guestfs -> string
-> string -> unit
 (** Run the filesystem-based export for all the formats listed in the
     [formats] set. *)
-- 
2.9.3

It will be useful also for APIs different than tar-out, so move it to
guestfsd.c, and add it a parameter to specify the function name that
invoked it.

This is mostly code motion.
---
 daemon/daemon.h   |  2 ++
 daemon/guestfsd.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 daemon/tar.c      | 58 +--------------------------------------------------
 3 files changed, 65 insertions(+), 57 deletions(-)

diff --git a/daemon/daemon.h b/daemon/daemon.h
index 8097bfc..793074d 100644
--- a/daemon/daemon.h
+++ b/daemon/daemon.h
@@ -147,6 +147,8 @@ extern int random_name (char *template);
 
 extern char *get_random_uuid (void);
 
+extern char *make_exclude_from_file (const char *function, char *const
*excludes);
+
 extern int asprintf_nowarn (char **strp, const char *fmt, ...);
 
 /*-- in names.c (auto-generated) --*/
diff --git a/daemon/guestfsd.c b/daemon/guestfsd.c
index 57e3886..a848fe1 100644
--- a/daemon/guestfsd.c
+++ b/daemon/guestfsd.c
@@ -1247,6 +1247,68 @@ get_random_uuid (void)
 
 }
 
+/**
+ * Turn list C<excludes> into a temporary file, and return a string
+ * containing the temporary file name.  Caller must unlink the file
+ * and free the string.
+ *
+ * C<function> is the function that invoked this helper, and it is
+ * used mainly for errors/debugging.
+ */
+char *
+make_exclude_from_file (const char *function, char *const *excludes)
+{
+  size_t i;
+  int fd;
+  char template[] = "/tmp/excludesXXXXXX";
+  char *ret;
+
+  fd = mkstemp (template);
+  if (fd == -1) {
+    reply_with_perror ("mkstemp");
+    return NULL;
+  }
+
+  for (i = 0; excludes[i] != NULL; ++i) {
+    if (strchr (excludes[i], '\n')) {
+      reply_with_error ("%s: excludes file patterns cannot contain \\n
character",
+                        function);
+      goto error;
+    }
+
+    if (xwrite (fd, excludes[i], strlen (excludes[i])) == -1 ||
+        xwrite (fd, "\n", 1) == -1) {
+      reply_with_perror ("write");
+      goto error;
+    }
+
+    if (verbose)
+      fprintf (stderr, "%s: adding excludes pattern '%s'\n",
+               function, excludes[i]);
+  }
+
+  if (close (fd) == -1) {
+    reply_with_perror ("close");
+    fd = -1;
+    goto error;
+  }
+  fd = -1;
+
+  ret = strdup (template);
+  if (ret == NULL) {
+    reply_with_perror ("strdup");
+    goto error;
+  }
+
+  return ret;
+
+ error:
+  if (fd >= 0)
+    close (fd);
+  unlink (template);
+  return NULL;
+}
+
 void
 cleanup_free_mountable (mountable_t *mountable)
 {
diff --git a/daemon/tar.c b/daemon/tar.c
index ea8b342..c23aa0a 100644
--- a/daemon/tar.c
+++ b/daemon/tar.c
@@ -268,62 +268,6 @@ do_txz_in (const char *dir)
   return do_tar_in (dir, "xz", 0, 0, 0);
 }
 
-/* Turn list 'excludes' into a temporary file, and return a string
- * containing the temporary file name.  Caller must unlink the file
- * and free the string.
- */
-static char *
-make_exclude_from_file (char *const *excludes)
-{
-  size_t i;
-  int fd;
-  char template[] = "/tmp/excludesXXXXXX";
-  char *ret;
-
-  fd = mkstemp (template);
-  if (fd == -1) {
-    reply_with_perror ("mkstemp");
-    return NULL;
-  }
-
-  for (i = 0; excludes[i] != NULL; ++i) {
-    if (strchr (excludes[i], '\n')) {
-      reply_with_error ("tar-out: excludes file patterns cannot contain
\\n character");
-      goto error;
-    }
-
-    if (xwrite (fd, excludes[i], strlen (excludes[i])) == -1 ||
-        xwrite (fd, "\n", 1) == -1) {
-      reply_with_perror ("write");
-      goto error;
-    }
-
-    if (verbose)
-      fprintf (stderr, "tar-out: adding excludes pattern
'%s'\n", excludes[i]);
-  }
-
-  if (close (fd) == -1) {
-    reply_with_perror ("close");
-    fd = -1;
-    goto error;
-  }
-  fd = -1;
-
-  ret = strdup (template);
-  if (ret == NULL) {
-    reply_with_perror ("strdup");
-    goto error;
-  }
-
-  return ret;
-
- error:
-  if (fd >= 0)
-    close (fd);
-  unlink (template);
-  return NULL;
-}
-
 /* Has one FileOut parameter. */
 /* Takes optional arguments, consult optargs_bitmask. */
 int
@@ -367,7 +311,7 @@ do_tar_out (const char *dir, const char *compress, int
numericowner,
     numericowner = 0;
 
   if ((optargs_bitmask & GUESTFS_TAR_OUT_EXCLUDES_BITMASK)) {
-    exclude_from_file = make_exclude_from_file (excludes);
+    exclude_from_file = make_exclude_from_file ("tar-out", excludes);
     if (!exclude_from_file)
       return -1;
   }
-- 
2.9.3

Introduce a new API to create a new squashfs filesystem out of a path
in the guest.  It can be configured to exclude paths based on patterns,
and to select which compression use for the filesystem.

The advantage of running mksquashfs directly in the appliance is that
ownerships are properly saved, as opposed to tar_out + local untar.
---
 appliance/packagelist.in |   2 +
 daemon/Makefile.am       |   1 +
 daemon/squashfs.c        | 150 +++++++++++++++++++++++++++++++++++++++++++++++
 generator/actions.ml     |  26 ++++++++
 gobject/Makefile.inc     |   2 +
 lib/MAX_PROC_NR          |   2 +-
 6 files changed, 182 insertions(+), 1 deletion(-)
 create mode 100644 daemon/squashfs.c

diff --git a/appliance/packagelist.in b/appliance/packagelist.in
index 1cf7d9d..8ed3afe 100644
--- a/appliance/packagelist.in
+++ b/appliance/packagelist.in
@@ -151,6 +151,7 @@ ifelse(SUSE,1,
   ntfsprogs
   ntfs-3g
   reiserfs
+  squashfs
   systemd
   vim
   xz
@@ -253,6 +254,7 @@ rsync
 scrub
 sed
 sleuthkit
+squashfs-tools
 strace
 syslinux
 tar
diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 3e301ad..5c4ae8e 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -138,6 +138,7 @@ guestfsd_SOURCES = \
 	sh.c \
 	sleep.c \
 	sleuthkit.c \
+	squashfs.c \
 	stat.c \
 	statvfs.c \
 	strings.c \
diff --git a/daemon/squashfs.c b/daemon/squashfs.c
new file mode 100644
index 0000000..c307776
--- /dev/null
+++ b/daemon/squashfs.c
@@ -0,0 +1,150 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "daemon.h"
+#include "actions.h"
+#include "optgroups.h"
+
+#define MAX_ARGS 64
+
+GUESTFSD_EXT_CMD(str_mksquashfs, mksquashfs);
+
+int
+optgroup_squashfs_available (void)
+{
+  return prog_exists (str_mksquashfs);
+}
+
+/* Takes optional arguments, consult optargs_bitmask. */
+int
+do_mksquashfs (const char *path, const char *compress, char *const *excludes)
+{
+  CLEANUP_FREE char *buf = NULL;
+  CLEANUP_UNLINK_FREE char *tmpfile = NULL;
+  CLEANUP_UNLINK_FREE char *exclude_from_file = NULL;
+  CLEANUP_FREE char *err = NULL;
+  CLEANUP_FREE char *buffer = NULL;
+  int r;
+  const char *argv[MAX_ARGS];
+  size_t i = 0;
+  int fd;
+  FILE *fp;
+
+  buf = sysroot_path (path);
+  if (buf == NULL) {
+    reply_with_perror ("malloc/sysroot_path");
+    return -1;
+  }
+
+  /* /var/tmp is used instead of /tmp, as /tmp is mounted as tmpfs
+   * and thus a newly created filesystem might not fit in memory.
+   */
+  tmpfile = strdup ("/var/tmp/squashfs.XXXXXX");
+  if (tmpfile == NULL) {
+    reply_with_perror ("strdup");
+    return -1;
+  }
+
+  fd = mkstemp (tmpfile);
+  if (fd == -1) {
+    reply_with_perror ("mkstemp");
+    return -1;
+  }
+  close (fd);
+
+  buffer = malloc (GUESTFS_MAX_CHUNK_SIZE);
+  if (buffer == NULL) {
+    reply_with_perror ("malloc/buffer");
+    return -1;
+  }
+
+  ADD_ARG (argv, i, str_mksquashfs);
+  ADD_ARG (argv, i, buf);
+  ADD_ARG (argv, i, tmpfile);
+  ADD_ARG (argv, i, "-noappend");
+  ADD_ARG (argv, i, "-root-becomes");
+  ADD_ARG (argv, i, buf);
+  ADD_ARG (argv, i, "-wildcards");
+  ADD_ARG (argv, i, "-no-recovery");
+
+  if (optargs_bitmask & GUESTFS_MKSQUASHFS_COMPRESS_BITMASK) {
+    ADD_ARG (argv, i, "-comp");
+    ADD_ARG (argv, i, compress);
+  }
+
+  if (optargs_bitmask & GUESTFS_MKSQUASHFS_EXCLUDES_BITMASK) {
+    exclude_from_file = make_exclude_from_file ("mksquashfs",
excludes);
+    if (!exclude_from_file)
+      return -1;
+
+    ADD_ARG (argv, i, "-ef");
+    ADD_ARG (argv, i, exclude_from_file);
+  }
+
+  ADD_ARG (argv, i, NULL);
+
+  r = commandv (NULL, &err, argv);
+  if (r == -1) {
+    reply_with_error ("%s: %s", path, err);
+    return -1;
+  }
+
+  fp = fopen (tmpfile, "r");
+  if (fp == NULL) {
+    reply_with_perror ("%s", tmpfile);
+    return -1;
+  }
+
+  /* Now we must send the reply message, before the file contents.  After
+   * this there is no opportunity in the protocol to send any error
+   * message back.  Instead we can only cancel the transfer.
+   */
+  reply (NULL, NULL);
+
+  while ((r = fread (buffer, 1, GUESTFS_MAX_CHUNK_SIZE, fp)) > 0) {
+    if (send_file_write (buffer, r) < 0) {
+      fclose (fp);
+      return -1;
+    }
+  }
+
+  if (ferror (fp)) {
+    fprintf (stderr, "fread: %s: %m\n", tmpfile);
+    send_file_end (1);		/* Cancel. */
+    fclose (fp);
+    return -1;
+  }
+
+  if (fclose (fp) != 0) {
+    fprintf (stderr, "fclose: %s: %m\n", tmpfile);
+    send_file_end (1);		/* Cancel. */
+    return -1;
+  }
+
+  if (send_file_end (0))	/* Normal end of file. */
+    return -1;
+
+  return 0;
+}
diff --git a/generator/actions.ml b/generator/actions.ml
index fd6cc9f..409f399 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -13292,6 +13292,32 @@ is removed." };
     shortdesc = "search the entries associated to the given inode";
     longdesc = "Internal function for find_inode." };
 
+  { defaults with
+    name = "mksquashfs"; added = (1, 35, 25);
+    style = RErr, [Pathname "path"; FileOut "filename"],
[OString "compress"; OStringList "excludes"];
+    proc_nr = Some 471;
+    optional = Some "squashfs";
+    cancellable = true;
+    shortdesc = "create a squashfs filesystem";
+    longdesc = "\
+Create a squashfs filesystem for the specified C<path>.
+
+The optional C<compress> flag controls compression.  If not given,
+then the output compressed using C<gzip>.  Otherwise one
+of the following strings may be given to select the compression
+type of the squashfs: C<gzip>, C<lzma>, C<lzo>, C<lz4>,
C<xz>.
+
+The other optional arguments are:
+
+=over 4
+
+=item C<excludes>
+
+A list of wildcards.  Files are excluded if they match any of the
+wildcards.
+
+=back" };
+
 ]
 
 (* Non-API meta-commands available only in guestfish.
diff --git a/gobject/Makefile.inc b/gobject/Makefile.inc
index 76482f7..48cb2ef 100644
--- a/gobject/Makefile.inc
+++ b/gobject/Makefile.inc
@@ -90,6 +90,7 @@ guestfs_gobject_headers= \
   include/guestfs-gobject/optargs-mke2fs.h \
   include/guestfs-gobject/optargs-mkfs.h \
   include/guestfs-gobject/optargs-mkfs_btrfs.h \
+  include/guestfs-gobject/optargs-mksquashfs.h \
   include/guestfs-gobject/optargs-mkswap.h \
   include/guestfs-gobject/optargs-mktemp.h \
   include/guestfs-gobject/optargs-mount_9p.h \
@@ -182,6 +183,7 @@ guestfs_gobject_sources= \
   src/optargs-mke2fs.c \
   src/optargs-mkfs.c \
   src/optargs-mkfs_btrfs.c \
+  src/optargs-mksquashfs.c \
   src/optargs-mkswap.c \
   src/optargs-mktemp.c \
   src/optargs-mount_9p.c \
diff --git a/lib/MAX_PROC_NR b/lib/MAX_PROC_NR
index 5f476b6..c305aa5 100644
--- a/lib/MAX_PROC_NR
+++ b/lib/MAX_PROC_NR
@@ -1 +1 @@
-470
+471
-- 
2.9.3

Implement the "squash" output format, i.e. a squashfs compressed
filesystem.

This was implemented in diskimage-builder upstream as
commit 9d13084c4183b63587e1f5e4b03395a8df6538f6.
---
 dib/Makefile.am               |  1 +
 dib/output_format_squashfs.ml | 39 +++++++++++++++++++++++++++++++++++++++
 dib/utils.ml                  |  1 +
 dib/virt-dib.pod              |  6 ++++++
 4 files changed, 47 insertions(+)
 create mode 100644 dib/output_format_squashfs.ml

diff --git a/dib/Makefile.am b/dib/Makefile.am
index 470fb52..6780ee2 100644
--- a/dib/Makefile.am
+++ b/dib/Makefile.am
@@ -31,6 +31,7 @@ formats = \
 	docker \
 	qcow2 \
 	raw \
+	squashfs \
 	tar \
 	tgz \
 	vhd
diff --git a/dib/output_format_squashfs.ml b/dib/output_format_squashfs.ml
new file mode 100644
index 0000000..333e711
--- /dev/null
+++ b/dib/output_format_squashfs.ml
@@ -0,0 +1,39 @@
+(* virt-dib
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_utils
+open Common_gettext.Gettext
+
+open Output_format
+
+let squashfs_check (g : Guestfs.guestfs) +  g#available [| "squashfs"
|]
+
+let squashfs_run_fs (g : Guestfs.guestfs) filename _ +  message
(f_"Compressing the image as squashfs");
+  g#mksquashfs ~excludes:[| "sys/*"; "proc/*";
"dev/*" |] ~compress:"xz"
+    "/" filename
+
+let fmt = {
+  defaults with
+    name = "squashfs";
+    check_appliance_prerequisites = Some squashfs_check;
+    run_on_filesystem = Some squashfs_run_fs;
+}
+
+let () = register_format fmt
diff --git a/dib/utils.ml b/dib/utils.ml
index 3775a41..da5e738 100644
--- a/dib/utils.ml
+++ b/dib/utils.ml
@@ -33,6 +33,7 @@ let current_arch ()    | arch -> arch
 
 let output_filename image_name = function
+  | "squashfs" -> image_name ^ ".squash"
   | fmt -> image_name ^ "." ^ fmt
 
 let log_filename () diff --git a/dib/virt-dib.pod b/dib/virt-dib.pod
index 2786050..7083e78 100644
--- a/dib/virt-dib.pod
+++ b/dib/virt-dib.pod
@@ -212,6 +212,12 @@ QEMU's qcow2.
 
 Raw disk format.
 
+=item C<squashfs>
+
+An squashfs filesystem, compressed with XZ.  This output format
+requires the C<squashfs> feature; see also
+L<guestfs(3)/AVAILABILITY>.
+
 =item C<tar>
 
 An uncompressed tarball.
-- 
2.9.3

On Tue, Feb 14, 2017 at 09:12:02AM +0100, Pino Toscano
wrote:
> Query awk for the list of environment variables, instead of trying to
> extract the list from the output of `env`: the old approach breaks when
> any of the environment variable contains more than one line.
> ---
>  dib/dib.ml | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/dib/dib.ml b/dib/dib.ml
> index 71b1f7f..54ea2ae 100644
> --- a/dib/dib.ml
> +++ b/dib/dib.ml
> @@ -289,7 +289,7 @@ if [ -n \"$user\" ]; then
>  fi
>  
>  if [ -z \"$preserve_env\" ]; then
> -  for envvar in `env | grep '^\\w' | cut -d= -f1`; do
> +  for envvar in `awk 'BEGIN{for (i in ENVIRON) {print i}}'`; do
FWIW you can do some really weird sh*t with environment variables,
such as creating ones whose name contains newlines.  That would break
the above code:

$ env $'A\nB=C' awk 'BEGIN{for (i in ENVIRON) {print i}}'

But probably not important enough to worry about, so ACK :-)

Rich.
>      case \"$envvar\" in
>        PATH | USER | USERNAME | HOSTNAME | TERM | LANG | HOME | SHELL |
LOGNAME ) ;;
>        BASH_FUNC_* ) unset -f $envvar ;;
> -- 
> 2.9.3
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs
-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v

On Tue, Feb 14, 2017 at 09:12:07AM +0100, Pino Toscano
wrote:
> Add a new 'excludes' optional argument to copy-out, passing it
straight
> to tar-out: this way it is possible to exclude files when extracting a
> directory from the guest.
> ---
>  generator/actions.ml | 16 ++++++++++++++--
>  gobject/Makefile.inc |  2 ++
>  lib/copy-in-out.c    | 13 ++++++++++---
>  3 files changed, 26 insertions(+), 5 deletions(-)
> 
> diff --git a/generator/actions.ml b/generator/actions.ml
> index 990eacb..fd6cc9f 100644
> --- a/generator/actions.ml
> +++ b/generator/actions.ml
> @@ -3437,8 +3437,9 @@ Wildcards cannot be used." };
>  
>    { defaults with
>      name = "copy_out"; added = (1, 29, 24);
> -    style = RErr, [Pathname "remotepath"; String
"localdir"], [];
> +    style = RErr, [Pathname "remotepath"; String
"localdir"], [OStringList "excludes"];
This change seems reasonable.  Should we:

(1) Bind other tar-out parameters?  It looks as if numericowner,
xattrs, selinux and acls would all be candidates.

(2) More importantly for this patch, keep the order of the options
compatible with tar-out?  This would allow us to pass the
optargs_bitmask straight through, if that's an advantage (it may not
be). The 'compressed' optarg for tar-out makes this a bit more
complicated.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
libguestfs lets you edit virtual machines.  Supports shell scripting,
bindings from many languages.  http://libguestfs.org

On Tue, Feb 14, 2017 at 09:12:10AM +0100, Pino Toscano
wrote:
> Introduce a new API to create a new squashfs filesystem out of a path
> in the guest.  It can be configured to exclude paths based on patterns,
> and to select which compression use for the filesystem.
> 
> The advantage of running mksquashfs directly in the appliance is that
> ownerships are properly saved, as opposed to tar_out + local untar.
I'm fairly sure this was proposed before as an API, but I cannot find
that right now.  Was rejected because mksquashfs doesn't support
streaming output.

However I'm fine with this, although perhaps there should be a warning
in the longdesc saying that there may be problems with large files.  I
think this API will break in the region of 3GB.

Rich.
> diff --git a/generator/actions.ml b/generator/actions.ml
> index fd6cc9f..409f399 100644
> --- a/generator/actions.ml
> +++ b/generator/actions.ml
> @@ -13292,6 +13292,32 @@ is removed." };
>      shortdesc = "search the entries associated to the given
inode";
>      longdesc = "Internal function for find_inode." };
>  
> +  { defaults with
> +    name = "mksquashfs"; added = (1, 35, 25);
> +    style = RErr, [Pathname "path"; FileOut
"filename"], [OString "compress"; OStringList
"excludes"];
> +    proc_nr = Some 471;
> +    optional = Some "squashfs";
> +    cancellable = true;
> +    shortdesc = "create a squashfs filesystem";
> +    longdesc = "\
> +Create a squashfs filesystem for the specified C<path>.
> +
> +The optional C<compress> flag controls compression.  If not given,
> +then the output compressed using C<gzip>.  Otherwise one
> +of the following strings may be given to select the compression
> +type of the squashfs: C<gzip>, C<lzma>, C<lzo>,
C<lz4>, C<xz>.
> +
> +The other optional arguments are:
> +
> +=over 4
> +
> +=item C<excludes>
> +
> +A list of wildcards.  Files are excluded if they match any of the
> +wildcards.
> +
-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
Fedora Windows cross-compiler. Compile Windows programs, test, and
build Windows installers. Over 100 libraries supported.
http://fedoraproject.org/wiki/MinGW

On Tue, Feb 14, 2017 at 09:12:11AM +0100, Pino Toscano
wrote:
> Implement the "squash" output format, i.e. a squashfs compressed
> filesystem.
Patch series looks reasonable, modulo the small comments I made to
specific patches.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
Fedora Windows cross-compiler. Compile Windows programs, test, and
build Windows installers. Over 100 libraries supported.
http://fedoraproject.org/wiki/MinGW