Libguestfs - Sep 2011 - [PATCH] New APIs: {compress, gzip, xz}-out, {compress, gzip, xz}-device-out.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming blog: http://rwmj.wordpress.com
Fedora now supports 80 OCaml packages (the OPEN alternative to F#)
http://cocan.org/getting_started_with_ocaml_on_red_hat_and_fedora
-------------- next part --------------
>From 1ff2c479a63574395903361bfb037e555c69ad26 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones at redhat.com>
Date: Wed, 28 Sep 2011 11:14:06 +0100
Subject: [PATCH] New APIs: {compress,gzip,xz}-out,
 {compress,gzip,xz}-device-out.

These APIs let you copy compressed files or devices out from the disk
image.

Compression is useful for large images which are mostly zeroes.  We
cannot currently do sparseness detection, and compression gives us a
form of zero detection for free.

Example usage:

$ guestfish -N fs gzip-device-out /dev/vda1 - > /tmp/vda1.gz
$ ll /tmp/vda1.gz
-rw-rw-r--. 1 rjones rjones 106822 Sep 28 11:16 /tmp/vda1.gz
$ file /tmp/vda1.gz
/tmp/vda1.gz: gzip compressed data, was "vda1", from Unix, last
modified: Wed Sep 28 11:16:54 2011
$ file -z /tmp/vda1.gz
/tmp/vda1.gz: Linux rev 1.0 ext2 filesystem data,
UUID=2592fc6a-ab3a-48be-a9f6-6920148b3bda (gzip compressed data, was
"vda1", from Unix, last modified: Wed Sep 28 11:16:54 2011)
---
 daemon/Makefile.am             |    1 +
 daemon/compress.c              |  141 ++++++++++++++++++++++++++++++++++++++++
 generator/generator_actions.ml |   59 +++++++++++++++++
 po/POTFILES.in                 |    1 +
 src/MAX_PROC_NR                |    2 +-
 5 files changed, 203 insertions(+), 1 deletions(-)
 create mode 100644 daemon/compress.c

diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 1664af0..e23ce86 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -94,6 +94,7 @@ guestfsd_SOURCES = \
 	checksum.c \
 	cmp.c \
 	command.c \
+	compress.c \
 	cpmv.c \
 	daemon.h \
 	dd.c \
diff --git a/daemon/compress.c b/daemon/compress.c
new file mode 100644
index 0000000..a63708d
--- /dev/null
+++ b/daemon/compress.c
@@ -0,0 +1,141 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2011 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "guestfs_protocol.h"
+#include "daemon.h"
+#include "actions.h"
+#include "optgroups.h"
+
+/* Has one FileOut parameter. */
+static int
+do_compressX_out (const char *file, const char *filter, int device)
+{
+  int r;
+  FILE *fp;
+  char *cmd;
+  char buf[GUESTFS_MAX_CHUNK_SIZE];
+
+  /* The command will look something like:
+   *   gzip -c /sysroot%s     # file
+   * or:
+   *   gzip -c %s             # device
+   * We have to quote the file or device name.
+   */
+  if (!device) {
+    if (asprintf_nowarn (&cmd, "%s %R", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  } else {
+    if (asprintf_nowarn (&cmd, "%s %Q", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  }
+
+  if (verbose)
+    fprintf (stderr, "%s\n", cmd);
+
+  fp = popen (cmd, "r");
+  if (fp == NULL) {
+    reply_with_perror ("%s", cmd);
+    free (cmd);
+    return -1;
+  }
+  free (cmd);
+
+  /* Now we must send the reply message, before the file contents.  After
+   * this there is no opportunity in the protocol to send any error
+   * message back.  Instead we can only cancel the transfer.
+   */
+  reply (NULL, NULL);
+
+  while ((r = fread (buf, 1, sizeof buf, fp)) > 0) {
+    if (send_file_write (buf, r) < 0) {
+      pclose (fp);
+      return -1;
+    }
+  }
+
+  if (ferror (fp)) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    pclose (fp);
+    return -1;
+  }
+
+  if (pclose (fp) != 0) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    return -1;
+  }
+
+  if (send_file_end (0))	/* Normal end of file. */
+    return -1;
+
+  return 0;
+}
+
+/* Has one FileOut parameter. */
+int
+do_compress_out (const char *file)
+{
+  return do_compressX_out (file, "compress -c", 0);
+}
+
+/* Has one FileOut parameter. */
+int
+do_gzip_out (const char *file)
+{
+  return do_compressX_out (file, "gzip -c", 0);
+}
+
+/* Has one FileOut parameter. */
+int
+do_xz_out (const char *file)
+{
+  return do_compressX_out (file, "xz -c", 0);
+}
+
+/* Has one FileOut parameter. */
+int
+do_compress_device_out (const char *device)
+{
+  return do_compressX_out (device, "compress -c", 1);
+}
+
+/* Has one FileOut parameter. */
+int
+do_gzip_device_out (const char *device)
+{
+  return do_compressX_out (device, "gzip -c", 1);
+}
+
+/* Has one FileOut parameter. */
+int
+do_xz_device_out (const char *device)
+{
+  return do_compressX_out (device, "xz -c", 1);
+}
diff --git a/generator/generator_actions.ml b/generator/generator_actions.ml
index c3d74f5..e4c31fb 100644
--- a/generator/generator_actions.ml
+++ b/generator/generator_actions.ml
@@ -6146,6 +6146,65 @@ C<path> does not exist, then a new file is created.
 
 See also C<guestfs_write>.");
 
+  ("compress_out", (RErr, [Pathname "file"; FileOut
"zfile"], []), 291, [],
+   [],
+   "output compressed file",
+   "\
+This command compresses C<file> and writes it out to the local
+file C<zfile>.
+
+For other forms of compression, see C<guestfs_gzip_out>,
C<guestfs_xz_out>.");
+
+  ("gzip_out", (RErr, [Pathname "file"; FileOut
"zfile"], []), 292, [],
+   [],
+   "output gzip-compressed file",
+   "\
+This command gzip-compresses C<file> and writes it out to the local
+file C<zfile>.
+
+For other forms of compression, see C<guestfs_compress_out>,
+C<guestfs_xz_out>.");
+
+  ("xz_out", (RErr, [Pathname "file"; FileOut
"zfile"], []), 293, [Optional "xz"],
+   [],
+   "output xz-compressed file",
+   "\
+This command xz-compresses C<file> and writes it out to the local
+file C<zfile>.
+
+For other forms of compression, see C<guestfs_compress_out>,
+C<guestfs_gzip_out>.");
+
+  ("compress_device_out", (RErr, [Device "device"; FileOut
"zdevice"], []), 294, [],
+   [],
+   "output compressed device",
+   "\
+This command compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+For other forms of compression, see C<guestfs_gzip_device_out>,
+C<guestfs_xz_device_out>.");
+
+  ("gzip_device_out", (RErr, [Device "device"; FileOut
"zdevice"], []), 295, [],
+   [],
+   "output gzip-compressed device",
+   "\
+This command gzip-compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+For other forms of compression, see C<guestfs_compress_device_out>,
+C<guestfs_xz_device_out>.");
+
+  ("xz_device_out", (RErr, [Device "device"; FileOut
"zdevice"], []), 296, [],
+   [],
+   "output xz-compressed device",
+   "\
+This command xz-compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+For other forms of compression, see C<guestfs_gzip_device_out>,
+C<guestfs_xz_device_out>.");
+
 ]
 
 let all_functions = non_daemon_functions @ daemon_functions
diff --git a/po/POTFILES.in b/po/POTFILES.in
index df54873..effc9ea 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,6 +11,7 @@ daemon/btrfs.c
 daemon/checksum.c
 daemon/cmp.c
 daemon/command.c
+daemon/compress.c
 daemon/cpmv.c
 daemon/dd.c
 daemon/debug.c
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 8408670..9530e04 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-290
+296
-- 
1.7.6

This updates the existing patch by removing 'compress'.  It turns out
this program isn't installed in the appliance, and in any case the
compress format is long obsolete.

Below are some timings.  Note that even for plain gzip compression
there is quite a large time penalty.  Virtio-serial / the libguestfs
protocol is quite efficient at copying large files around
(75 megabytes/sec, only a fraction slower than 'dd' on the host).

There is no multithread support in gzip or xz at the moment, so adding
SMP support to the appliance probably won't make much difference,
although it's worth adding it anyway.

I might play with snappy and a simple RLE-based compressor next.

Rich.

# Newly created, empty, 100MB ext2 filesystem:

$ ./run ./fish/guestfish -N fs time download /dev/vda1 >(wc -c)
elapsed time: 0.65 seconds
104792576

$ ./run ./fish/guestfish -N fs time gzip-device-out /dev/vda1 >(wc -c)
elapsed time: 1.08 seconds
106799

$ ./run ./fish/guestfish -N fs time xz-device-out /dev/vda1 >(wc -c)
elapsed time: 9.83 seconds
18168

# On a real filesystem that is too large to fit into cache:
# Filesystem                                Size       Used  Available  Use%
# F16x64:/dev/vg_f16x64/lv_root             5.5G       3.2G       2.2G   59%

$ ./run ./fish/guestfish --ro -a /dev/vg_pin/F16x64 run : time download
/dev/vg_f16x64/lv_root >(wc -c)
elapsed time: 78.74 seconds
5972688896

$ ./run ./fish/guestfish --ro -a /dev/vg_pin/F16x64 run : time gzip-device-out
/dev/vg_f16x64/lv_root >(wc -c)
elapsed time: 274.11 seconds
1927316373

[xz case took more than 10 minutes, I didn't wait for it to finish]

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine.  Supports Linux and Windows.
http://et.redhat.com/~rjones/virt-df/
-------------- next part --------------
>From 8e4eb015aa9a9dbe2bdf676bc591cc158dd90afe Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones at redhat.com>
Date: Wed, 28 Sep 2011 11:14:06 +0100
Subject: [PATCH] New APIs: gzip-out, xz-out, gzip-device-out, xz-device-out.

These APIs let you copy compressed files or devices out from the disk
image.

Compression is useful for large images which are mostly zeroes.  We
cannot currently do sparseness detection, and compression gives us a
form of zero detection for free.

Example usage:

$ guestfish -N fs gzip-device-out /dev/vda1 - > /tmp/vda1.gz
$ ll /tmp/vda1.gz
-rw-rw-r--. 1 rjones rjones 106822 Sep 28 11:16 /tmp/vda1.gz
$ file /tmp/vda1.gz
/tmp/vda1.gz: gzip compressed data, was "vda1", from Unix, last
modified: Wed Sep 28 11:16:54 2011
$ file -z /tmp/vda1.gz
/tmp/vda1.gz: Linux rev 1.0 ext2 filesystem data,
UUID=2592fc6a-ab3a-48be-a9f6-6920148b3bda (gzip compressed data, was
"vda1", from Unix, last modified: Wed Sep 28 11:16:54 2011)
---
 daemon/Makefile.am             |    1 +
 daemon/compress.c              |  127 ++++++++++++++++++++++++++++++++++++++++
 generator/generator_actions.ml |   36 +++++++++++
 po/POTFILES.in                 |    1 +
 src/MAX_PROC_NR                |    2 +-
 5 files changed, 166 insertions(+), 1 deletions(-)
 create mode 100644 daemon/compress.c

diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 1664af0..e23ce86 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -94,6 +94,7 @@ guestfsd_SOURCES = \
 	checksum.c \
 	cmp.c \
 	command.c \
+	compress.c \
 	cpmv.c \
 	daemon.h \
 	dd.c \
diff --git a/daemon/compress.c b/daemon/compress.c
new file mode 100644
index 0000000..0c54e41
--- /dev/null
+++ b/daemon/compress.c
@@ -0,0 +1,127 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2011 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "guestfs_protocol.h"
+#include "daemon.h"
+#include "actions.h"
+#include "optgroups.h"
+
+/* Has one FileOut parameter. */
+static int
+do_compressX_out (const char *file, const char *filter, int device)
+{
+  int r;
+  FILE *fp;
+  char *cmd;
+  char buf[GUESTFS_MAX_CHUNK_SIZE];
+
+  /* The command will look something like:
+   *   gzip -c /sysroot%s     # file
+   * or:
+   *   gzip -c %s             # device
+   * We have to quote the file or device name.
+   */
+  if (!device) {
+    if (asprintf_nowarn (&cmd, "%s %R", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  } else {
+    if (asprintf_nowarn (&cmd, "%s %Q", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  }
+
+  if (verbose)
+    fprintf (stderr, "%s\n", cmd);
+
+  fp = popen (cmd, "r");
+  if (fp == NULL) {
+    reply_with_perror ("%s", cmd);
+    free (cmd);
+    return -1;
+  }
+  free (cmd);
+
+  /* Now we must send the reply message, before the file contents.  After
+   * this there is no opportunity in the protocol to send any error
+   * message back.  Instead we can only cancel the transfer.
+   */
+  reply (NULL, NULL);
+
+  while ((r = fread (buf, 1, sizeof buf, fp)) > 0) {
+    if (send_file_write (buf, r) < 0) {
+      pclose (fp);
+      return -1;
+    }
+  }
+
+  if (ferror (fp)) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    pclose (fp);
+    return -1;
+  }
+
+  if (pclose (fp) != 0) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    return -1;
+  }
+
+  if (send_file_end (0))	/* Normal end of file. */
+    return -1;
+
+  return 0;
+}
+
+/* Has one FileOut parameter. */
+int
+do_gzip_out (const char *file)
+{
+  return do_compressX_out (file, "gzip -c", 0);
+}
+
+/* Has one FileOut parameter. */
+int
+do_xz_out (const char *file)
+{
+  return do_compressX_out (file, "xz -c", 0);
+}
+
+/* Has one FileOut parameter. */
+int
+do_gzip_device_out (const char *device)
+{
+  return do_compressX_out (device, "gzip -c", 1);
+}
+
+/* Has one FileOut parameter. */
+int
+do_xz_device_out (const char *device)
+{
+  return do_compressX_out (device, "xz -c", 1);
+}
diff --git a/generator/generator_actions.ml b/generator/generator_actions.ml
index c3d74f5..b5f2bd5 100644
--- a/generator/generator_actions.ml
+++ b/generator/generator_actions.ml
@@ -6146,6 +6146,42 @@ C<path> does not exist, then a new file is created.
 
 See also C<guestfs_write>.");
 
+  ("gzip_out", (RErr, [Pathname "file"; FileOut
"zfile"], []), 291, [],
+   [],
+   "output gzip-compressed file",
+   "\
+This command gzip-compresses C<file> and writes it out to the local
+file C<zfile>.
+
+For other forms of compression, see C<guestfs_xz_out>.");
+
+  ("xz_out", (RErr, [Pathname "file"; FileOut
"zfile"], []), 292, [Optional "xz"],
+   [],
+   "output xz-compressed file",
+   "\
+This command xz-compresses C<file> and writes it out to the local
+file C<zfile>.
+
+For other forms of compression, see C<guestfs_gzip_out>.");
+
+  ("gzip_device_out", (RErr, [Device "device"; FileOut
"zdevice"], []), 293, [],
+   [],
+   "output gzip-compressed device",
+   "\
+This command gzip-compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+For other forms of compression, see C<guestfs_xz_device_out>.");
+
+  ("xz_device_out", (RErr, [Device "device"; FileOut
"zdevice"], []), 294, [],
+   [],
+   "output xz-compressed device",
+   "\
+This command xz-compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+For other forms of compression, see C<guestfs_gzip_device_out>.");
+
 ]
 
 let all_functions = non_daemon_functions @ daemon_functions
diff --git a/po/POTFILES.in b/po/POTFILES.in
index df54873..effc9ea 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,6 +11,7 @@ daemon/btrfs.c
 daemon/checksum.c
 daemon/cmp.c
 daemon/command.c
+daemon/compress.c
 daemon/cpmv.c
 daemon/dd.c
 daemon/debug.c
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 8408670..26f42e6 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-290
+294
-- 
1.7.6