Libguestfs - Aug 2013 - [PATCH v2 0/3 supermin] URPMI & xz support.

Joseph,

Please try my modified versions of these patches.  These are
compile-tested on Fedora and they don't break any existing
functionality, but I don't have either urpmi nor a statically-linked
xz so I cannot fully test them.

I have also fixed detection of zlib (2/3).

Rich.

From: Joseph Wang <joequant@gmail.com>

- RWMJ: Some minor fixes, and update dependencies.
---
 configure.ac              |   7 ++-
 src/.depend               |   2 +
 src/Makefile.am           |   2 +
 src/config.ml.in          |   1 +
 src/supermin_urpmi_rpm.ml | 132 ++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 143 insertions(+), 1 deletion(-)
 create mode 100644 src/supermin_urpmi_rpm.ml

diff --git a/configure.ac b/configure.ac
index 0c435ca..c190e2c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -68,11 +68,16 @@ fi
 AM_CONDITIONAL(HAVE_PERLDOC,[test "$perldoc" != "no"])
 
 dnl For yum-rpm handler.
-AC_CHECK_PROG(ZYPPER,[zypper],[zypper],[no])
 AC_CHECK_PROG(YUM,[yum],[yum],[no])
 AC_CHECK_PROG(RPM,[rpm],[rpm],[no])
 AC_CHECK_PROG(YUMDOWNLOADER,[yumdownloader],[yumdownloader],[no])
 
+dnl For Zypper handler.
+AC_CHECK_PROG(ZYPPER,[zypper],[zypper],[no])
+
+dnl For URPMI handler.
+AC_CHECK_PROG(URPMI,[urpmi],[urpmi],[no], [$PATH$PATH_SEPARATOR/usr/sbin])
+
 dnl For Debian handler.
 AC_CHECK_PROG(APTITUDE,[aptitude],[aptitude],[no])
 AC_CHECK_PROG(APT_CACHE,[apt-cache],[apt-cache],[no])
diff --git a/src/.depend b/src/.depend
index 8342f31..c1a9147 100644
--- a/src/.depend
+++ b/src/.depend
@@ -12,6 +12,8 @@ supermin_package_handlers.cmo: supermin_utils.cmi
supermin_cmdline.cmi supermin_
 supermin_package_handlers.cmx: supermin_utils.cmx supermin_cmdline.cmx
supermin_package_handlers.cmi
 supermin_pacman.cmo: supermin_utils.cmi supermin_package_handlers.cmi
supermin_cmdline.cmi config.cmo
 supermin_pacman.cmx: supermin_utils.cmx supermin_package_handlers.cmx
supermin_cmdline.cmx config.cmx
+supermin_urpmi_rpm.cmo: supermin_utils.cmi supermin_package_handlers.cmi
supermin_cmdline.cmi config.cmo
+supermin_urpmi_rpm.cmx: supermin_utils.cmx supermin_package_handlers.cmx
supermin_cmdline.cmx config.cmx
 supermin_utils.cmi:
 supermin_utils.cmo: supermin_cmdline.cmi supermin_utils.cmi
 supermin_utils.cmx: supermin_cmdline.cmx supermin_utils.cmi
diff --git a/src/Makefile.am b/src/Makefile.am
index b0a6723..fe82000 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -28,6 +28,7 @@ EXTRA_DIST = \
 	supermin_package_handlers.mli \
 	supermin_package_handlers.ml \
 	supermin_pacman.ml \
+	supermin_urpmi_rpm.ml \
 	supermin_utils.mli \
 	supermin_utils.ml \
 	supermin_yum_rpm.ml \
@@ -50,6 +51,7 @@ endif
 
 SOURCES += \
 	supermin_yum_rpm.ml \
+	supermin_urpmi_rpm.ml \
 	supermin_debian.ml \
 	supermin_pacman.ml \
 	supermin.ml
diff --git a/src/config.ml.in b/src/config.ml.in
index c3e7672..a814f50 100644
--- a/src/config.ml.in
+++ b/src/config.ml.in
@@ -21,6 +21,7 @@ let package_name = "@PACKAGE_NAME@"
 let package_version = "@PACKAGE_VERSION@"
 let zypper = "@ZYPPER@"
 let yum = "@YUM@"
+let urpmi = "@URPMI@"
 let rpm = "@RPM@"
 let yumdownloader = "@YUMDOWNLOADER@"
 let aptitude = "@APTITUDE@"
diff --git a/src/supermin_urpmi_rpm.ml b/src/supermin_urpmi_rpm.ml
new file mode 100644
index 0000000..a598ef5
--- /dev/null
+++ b/src/supermin_urpmi_rpm.ml
@@ -0,0 +1,132 @@
+(* supermin 4
+ * Copyright (C) 2009-2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *)
+
+(* URPMI support. *)
+
+open Unix
+open Printf
+
+open Supermin_package_handlers
+open Supermin_utils
+open Supermin_cmdline
+
+(* Create a temporary directory for use by all the functions in this file. *)
+let tmpdir = tmpdir ()
+
+let urpmi_rpm_detect () +  (file_exists "/etc/mageia-release")
&&
+    Config.urpmi <> "no" && Config.rpm <>
"no"
+
+let urpmi_rpm_init () +  if use_installed then
+    failwith "urpmi_rpm driver doesn't support --use-installed"
+
+let urpmi_rpm_resolve_dependencies_and_download names mode +  if mode =
PkgNamesOnly then (
+    eprintf "supermin: urpmi-rpm: --names-only flag is not
implemented\n";
+    exit 1
+  );
+  let cmd = sprintf "urpmq -rd --whatprovides --sources %s"
+    (String.concat " " names) in
+  let lines = run_command_get_lines cmd in
+  (* Return list of package filenames. *)
+  let g x +     (Filename.concat tmpdir (Filename.basename x)) in
+  let f x +      let cmd = sprintf "curl %s -o %s" x (g x) in
+        run_command cmd in
+  List.iter f lines;
+  let uf res e = if List.mem e res then res else e::res in
+  List.fold_left uf [] (List.map g lines)
+
+let rec urpmi_rpm_list_files pkg +  (* Run rpm -qlp with some extra magic. *)
+  let cmd +    sprintf "rpm -q --qf '[%%{FILENAMES}
%%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s"
+      pkg in
+  let lines = run_command_get_lines cmd in
+
+  let files +    filter_map (
+      fun line ->
+        match string_split " " line with
+        | [filename; flags; mode; size] ->
+            let test_flag = String.contains flags in
+            let mode = int_of_string mode in
+	    let size = int_of_string size in
+            if test_flag 'd' then None  (* ignore documentation *)
+            else
+              Some (filename, {
+                      ft_dir = mode land 0o40000 <> 0;
+                      ft_ghost = test_flag 'g'; ft_config = test_flag
'c';
+                      ft_mode = mode; ft_size = size;
+                    })
+        | _ ->
+            eprintf "supermin: bad output from rpm command:
'%s'" line;
+            exit 1
+    ) lines in
+
+  (* I've never understood why the base packages like 'filesystem'
don't
+   * contain any /dev nodes at all.  This leaves every program that
+   * bootstraps RPMs to create a varying set of device nodes themselves.
+   * This collection was copied from mock/backend.py.
+   *)
+  let files +    let b = Filename.basename pkg in
+    if string_prefix "filesystem-" b then (
+      let dirs = [ "/proc"; "/sys"; "/dev";
"/dev/pts"; "/dev/shm";
+                   "/dev/mapper" ] in
+      let dirs +        List.map (fun name ->
+                    name, { ft_dir = true; ft_ghost = false;
+                            ft_config = false; ft_mode = 0o40755;
+			    ft_size = 0 }) dirs in
+      let devs = [ "/dev/null"; "/dev/full";
"/dev/zero"; "/dev/random";
+                   "/dev/urandom"; "/dev/tty";
"/dev/console";
+                   "/dev/ptmx"; "/dev/stdin";
"/dev/stdout"; "/dev/stderr" ] in
+      (* No need to set the mode because these will go into hostfiles. *)
+      let devs +        List.map (fun name ->
+                    name, { ft_dir = false; ft_ghost = false;
+                            ft_config = false; ft_mode = 0o644;
+			    ft_size = 0 }) devs in
+      dirs @ devs @ files
+    ) else files in
+
+  files
+
+let urpmi_rpm_get_file_from_package pkg file +  debug "extracting %s from
%s ..." file (Filename.basename pkg);
+
+  let outfile = tmpdir // file in
+  let cmd +    sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio
--quiet -id .%s)"
+      (Filename.quote pkg) (Filename.quote tmpdir) (Filename.quote file) in
+  run_command cmd;
+  outfile
+
+let () +  let ph = {
+    ph_detect = urpmi_rpm_detect;
+    ph_init = urpmi_rpm_init;
+    ph_resolve_dependencies_and_download +     
urpmi_rpm_resolve_dependencies_and_download;
+    ph_list_files = urpmi_rpm_list_files;
+    ph_get_file_from_package = urpmi_rpm_get_file_from_package;
+  } in
+  register_package_handler "urpmi-rpm" ph
-- 
1.8.3.1

From: "Richard W.M. Jones" <rjones@redhat.com>

---
 README             |  2 +-
 configure.ac       | 31 ++++++++++++++++++++++++++++++-
 helper/Makefile.am |  1 +
 helper/init.c      |  8 ++++----
 4 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/README b/README
index 6542471..3461d9d 100644
--- a/README
+++ b/README
@@ -68,7 +68,7 @@ are building:
   qemu >= 0.13
   kernel >= 2.6.36
 
-  zlib - if your kernel uses gzipped modules
+  zlib (statically linked) - if your kernel uses gzipped modules
 
 Building and installing
 -----------------------
diff --git a/configure.ac b/configure.ac
index c190e2c..b3144a4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -102,7 +102,36 @@ dnl For ArchLinux handler.
 AC_CHECK_PROG(PACMAN,[pacman],[pacman],[no])
 
 dnl Support for gzipped kernel modules.
-AC_CHECK_LIB([z],[gzopen])
+AC_CHECK_HEADER([zlib.h],[
+    AC_CHECK_LIB([z],[gzopen],[
+        AC_MSG_CHECKING([for gzip static library])
+        old_CFLAGS="$CFLAGS"
+        old_LDFLAGS="$LDFLAGS"
+        old_LIBS="$LIBS"
+        CFLAGS="$CFLAGS -static"
+        LDFLAGS="$LDFLAGS -static"
+        LIBS="$LIBS -lz"
+        AC_LINK_IFELSE([
+            #include <stdio.h>
+            #include <stdlib.h>
+            #include <zlib.h>
+            int main () { gzFile g = gzopen ("test", "rb");
exit (g ? 1 : 0); }
+        ],[
+            zlib_static=yes
+            ZLIB_STATIC_LIBS="-lz"
+            AC_MSG_RESULT([yes])
+        ],[
+            AC_MSG_RESULT([no])
+        ])
+        CFLAGS="$old_CFLAGS"
+        LDFLAGS="$old_LDFLAGS"
+        LIBS="$old_LIBS"
+    ])
+])
+if test "x$zlib_static" = "xyes"; then
+    AC_DEFINE([HAVE_ZLIB],[1],[Define if you have static zlib])
+    AC_SUBST([ZLIB_STATIC_LIBS])
+fi
 
 dnl mke2fs.
 AC_PATH_PROG([MKE2FS],[mke2fs],[no],
diff --git a/helper/Makefile.am b/helper/Makefile.am
index 55826d7..de32299 100644
--- a/helper/Makefile.am
+++ b/helper/Makefile.am
@@ -43,6 +43,7 @@ noinst_PROGRAMS = init
 init_SOURCES = init.c
 init_CFLAGS = -static
 init_LDFLAGS = -static
+init_LDADD = $(ZLIB_STATIC_LIBS)
 
 CLEANFILES = ext2init.S
 
diff --git a/helper/init.c b/helper/init.c
index c53bf9d..4c10bff 100644
--- a/helper/init.c
+++ b/helper/init.c
@@ -41,7 +41,7 @@
 
 #include <asm/unistd.h>
 
-#ifdef HAVE_LIBZ
+#ifdef HAVE_ZLIB
 #include <zlib.h>
 #endif
 
@@ -97,7 +97,7 @@ main ()
   print_uptime ();
   fprintf (stderr, "supermin: ext2 mini initrd starting up: "
            PACKAGE_VERSION
-#ifdef HAVE_LIBZ
+#ifdef HAVE_ZLIB
            " zlib"
 #endif
            "\n");
@@ -268,7 +268,7 @@ insmod (const char *filename)
   if (verbose)
     fprintf (stderr, "supermin: internal insmod %s\n", filename);
 
-#ifdef HAVE_LIBZ
+#ifdef HAVE_ZLIB
   gzFile gzfp = gzopen (filename, "rb");
   int capacity = 64*1024;
   char *buf = (char *) malloc (capacity);
@@ -328,7 +328,7 @@ insmod (const char *filename)
      */
   }
 
-#ifdef HAVE_LIBZ
+#ifdef HAVE_ZLIB
   free (buf);
 #endif
 }
-- 
1.8.3.1

From: Joseph Wang <joequant@gmail.com>

RWMJ:

 - Whitespace fixes.

 - Detect static-linked xz.
---
 README        |  2 ++
 configure.ac  | 32 ++++++++++++++++++++
 helper/init.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/README b/README
index 3461d9d..61eb953 100644
--- a/README
+++ b/README
@@ -70,6 +70,8 @@ are building:
 
   zlib (statically linked) - if your kernel uses gzipped modules
 
+  xz (statically linked) - if your kernel uses xz-compressed modules
+
 Building and installing
 -----------------------
 
diff --git a/configure.ac b/configure.ac
index b3144a4..8e3d64d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -133,6 +133,38 @@ if test "x$zlib_static" = "xyes"; then
     AC_SUBST([ZLIB_STATIC_LIBS])
 fi
 
+dnl Support for xzed kernel modules.
+AC_CHECK_HEADER([lzma.h],[
+    AC_CHECK_LIB([lzma],[lzma_code],[
+        AC_MSG_CHECKING([for xz static library])
+        old_CFLAGS="$CFLAGS"
+        old_LDFLAGS="$LDFLAGS"
+        old_LIBS="$LIBS"
+        CFLAGS="$CFLAGS -static"
+        LDFLAGS="$LDFLAGS -static"
+        LIBS="$LIBS -llzma"
+        AC_LINK_IFELSE([
+            #include <stdio.h>
+            #include <stdlib.h>
+            #include <lzma.h>
+            int main () { lzma_stream s = LZMA_STREAM_INIT; exit (s ? 1 : 0); }
+        ],[
+            lzma_static=yes
+            LZMA_STATIC_LIBS="-llzma"
+            AC_MSG_RESULT([yes])
+        ],[
+            AC_MSG_RESULT([no])
+        ])
+        CFLAGS="$old_CFLAGS"
+        LDFLAGS="$old_LDFLAGS"
+        LIBS="$old_LIBS"
+    ])
+])
+if test "x$lzma_static" = "xyes"; then
+    AC_DEFINE([HAVE_LZMA],[1],[Define if you have static lzma])
+    AC_SUBST([LZMA_STATIC_LIBS])
+fi
+
 dnl mke2fs.
 AC_PATH_PROG([MKE2FS],[mke2fs],[no],
              [$PATH$PATH_SEPARATOR/sbin$PATH_SEPARATOR])
diff --git a/helper/init.c b/helper/init.c
index 4c10bff..7f096ec 100644
--- a/helper/init.c
+++ b/helper/init.c
@@ -45,6 +45,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef HAVE_LZMA
+#include <lzma.h>
+#endif
+
 /* Maximum time to wait for the root device to appear (seconds).
  *
  * On slow machines with lots of disks (Koji running the 255 disk test
@@ -100,6 +104,9 @@ main ()
 #ifdef HAVE_ZLIB
            " zlib"
 #endif
+#ifdef HAVE_LZMA
+           " xz"
+#endif
            "\n");
 
   read_cmdline ();
@@ -260,6 +267,20 @@ main ()
   exit (EXIT_FAILURE);
 }
 
+#if HAVE_LZMA
+static int
+ends_with (const char *str, const char *suffix)
+{
+  if (!str || !suffix)
+    return 0;
+  size_t lenstr = strlen (str);
+  size_t lensuffix = strlen (suffix);
+  if (lensuffix >  lenstr)
+    return 0;
+  return strncmp (str + lenstr - lensuffix, suffix, lensuffix) == 0;
+}
+#endif
+
 static void
 insmod (const char *filename)
 {
@@ -269,15 +290,81 @@ insmod (const char *filename)
     fprintf (stderr, "supermin: internal insmod %s\n", filename);
 
 #ifdef HAVE_ZLIB
-  gzFile gzfp = gzopen (filename, "rb");
   int capacity = 64*1024;
   char *buf = (char *) malloc (capacity);
   int tmpsize = 8 * 1024;
   char tmp[tmpsize];
   int num;
 
+  errno = 0;
   size = 0;
 
+#ifdef HAVE_LZMA
+  if (ends_with(filename, ".xz")) {
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX,
+          LZMA_CONCATENATED);
+    if (verbose)
+      fprintf (stderr, "supermin: running xz\n");
+    FILE *fd = fopen (filename, "r");
+    if (!fd) {
+      perror("popen failed");
+      exit (EXIT_FAILURE);
+    }
+    char tmp_out[tmpsize];
+    strm.avail_in = 0;
+    strm.next_out = tmp_out;
+    strm.avail_out = tmpsize;
+
+    lzma_action action = LZMA_RUN;
+
+    while (1) {
+      if (strm.avail_in == 0) {
+	strm.next_in = tmp;
+	strm.avail_in = fread(tmp, 1, tmpsize, fd);
+
+	if (ferror(fd)) {
+	  // POSIX says that fread() sets errno if
+	  // an error occurred. ferror() doesn't
+	  // touch errno.
+	  perror("Error reading input file");
+	  exit (EXIT_FAILURE);
+	}
+	if (feof(fd)) action = LZMA_FINISH;
+      }
+
+      ret = lzma_code(&strm, action);
+
+      // Write and check write error before checking decoder error.
+      // This way as much data as possible gets written to output
+      // even if decoder detected an error.
+      if (strm.avail_out == 0 || ret != LZMA_OK) {
+          const size_t num =  tmpsize - strm.avail_out;
+          if (num > capacity) {
+	        buf = (char*) realloc (buf, size*2);
+	        capacity = size;
+          }
+          memcpy (buf+size, tmp_out, num);
+          capacity -= num;
+          size += num;
+          strm.next_out = tmp_out;
+          strm.avail_out = tmpsize;
+      }
+      if (ret != LZMA_OK) {
+	if (ret == LZMA_STREAM_END) {
+	    break;
+	} else {
+	 perror("internal error");
+        exit(EXIT_FAILURE);
+	}
+     }
+    }
+    fclose (fd);
+    if (verbose)
+      fprintf (stderr, "done with xz %d read\n", size);
+  } else {
+#endif
+  gzFile gzfp = gzopen (filename, "rb");
   if (gzfp == NULL) {
     fprintf (stderr, "insmod: gzopen failed: %s", filename);
     exit (EXIT_FAILURE);
@@ -296,6 +383,10 @@ insmod (const char *filename)
     exit (EXIT_FAILURE);
   }
   gzclose (gzfp);
+#ifdef HAVE_LZMA
+}
+#endif
+
 #else
   int fd = open (filename, O_RDONLY);
   if (fd == -1) {
-- 
1.8.3.1

I had to make some changes to the xz patch.

The code that was used for conftest had to be modified and I needed to add
lzma to the link line.  The urpmi patch worked fine.


On Mon, Aug 19, 2013 at 4:43 PM, Richard W.M. Jones
<rjones@redhat.com>wrote:
> Joseph,
>
> Please try my modified versions of these patches.  These are
> compile-tested on Fedora and they don't break any existing
> functionality, but I don't have either urpmi nor a statically-linked
> xz so I cannot fully test them.
>
> I have also fixed detection of zlib (2/3).
>
> Rich.
>
>

On Wed, Aug 21, 2013 at 08:39:32PM +0800, Joseph Wang
wrote:
> I had to make some changes to the xz patch.
> 
> The code that was used for conftest had to be modified and I needed to add
> lzma to the link line.  The urpmi patch worked fine.
Thanks, I've now pushed both.  I made another small change
to the xz patch because &s would always be != NULL.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v