Libguestfs - Sep 2011 - [PATCH] febootstrap: Use contents of installed Debian packages instead of downloading and unpacking them.

---
 src/febootstrap_debian.ml |   65 +++++++++++++++++++++++++++++++++++++-------
 1 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml
index 23f3593..bb10f15 100644
--- a/src/febootstrap_debian.ml
+++ b/src/febootstrap_debian.ml
@@ -28,6 +28,9 @@ open Febootstrap_cmdline
 (* Create a temporary directory for use by all the functions in this file. *)
 let tmpdir = tmpdir ()
 
+let installed_pkgs +  run_command_get_lines "dpkg-query --show
--showformat='${Package}\\n'"
+
 let debian_detect ()    file_exists "/etc/debian_version" &&
     Config.aptitude <> "no" && Config.apt_cache
<> "no" && Config.dpkg <> "no"
@@ -51,18 +54,29 @@ let rec debian_resolve_dependencies_and_download names      
not (List.exists (fun re -> Str.string_match re name 0) excludes)
     ) pkgs in
 
+  let present_pkgs, download_pkgs = List.partition (
+    fun pkg -> List.exists ((=) pkg) installed_pkgs
+  ) pkgs in
+
+  debug "wanted packages (present / download): %s / %s\n"
+    (String.concat " " present_pkgs)
+    (String.concat " " download_pkgs);
+
   (* Download the packages. *)
-  let cmd -    sprintf "umask 0000; cd %s && %s download %s"
-      (Filename.quote tmpdir)
-      Config.aptitude
-      (String.concat " " (List.map Filename.quote pkgs)) in
-  run_command cmd;
+  if (List.length download_pkgs > 0)
+  then (
+    let cmd +      sprintf "umask 0000; cd %s && %s download
%s"
+        (Filename.quote tmpdir)
+        Config.aptitude
+        (String.concat " " (List.map Filename.quote download_pkgs))
in
+    run_command cmd
+  );
 
   (* Find out what aptitude downloaded. *)
   let files = Sys.readdir tmpdir in
 
-  let pkgs = List.map (
+  let download_pkgs = List.map (
     fun pkg ->
       (* Look for 'pkg_*.deb' in the list of files. *)
       let pre = pkg ^ "_" in
@@ -79,9 +93,9 @@ let rec debian_resolve_dependencies_and_download names  	exit
1
       with
 	  Exit -> !r
-  ) pkgs in
+  ) download_pkgs in
 
-  List.sort compare pkgs
+  List.sort compare (List.append present_pkgs download_pkgs)
 
 (* On Ubuntu 10.04 LTS, apt-cache depends --recurse is broken.  It
  * doesn't return the full list of dependencies.  Therefore recurse
@@ -106,7 +120,7 @@ and workaround_broken_apt_cache_depends_recurse names   
else
     names
 
-let debian_list_files pkg +let debian_list_files_downloaded pkg    debug
"unpacking %s ..." pkg;
 
   (* We actually need to extract the file in order to get the
@@ -147,9 +161,38 @@ let debian_list_files pkg  
   files
 
+let debian_list_files_installed pkg +  debug "using installed package %s
..." pkg;
+  let cmd = sprintf "dpkg-query --listfiles %s" pkg in
+  let lines = run_command_get_lines cmd in
+  (* filter out lines not directly describing fs objects such as
+     "package diverts others to: /path/to/..." *)
+  let lines = List.filter (
+    fun l -> l.[0] = '/' && l.[1] != '.'
+  ) lines in
+  let files = List.map (
+    fun path ->
+      let statbuf = lstat path in
+      let is_dir = statbuf.st_kind = S_DIR in
+      let config = statbuf.st_kind = S_REG && string_prefix
"/etc/" path in
+      let mode = statbuf.st_perm in
+      (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode;
+	       ft_ghost = false; ft_size = statbuf.st_size })
+  ) lines in
+  files
+
+let debian_list_files pkg +  if List.exists ((=) pkg) installed_pkgs then
+    debian_list_files_installed pkg
+  else
+    debian_list_files_downloaded pkg
+
 (* Easy because we already unpacked the archive above. *)
 let debian_get_file_from_package pkg file -  tmpdir // pkg ^ ".d" //
file
+  if List.exists (fun p -> p = pkg) installed_pkgs then
+    file
+  else
+    tmpdir // pkg ^ ".d" // file
 
 let ()    let ph = {
-- 
1.7.5.4

On Mon, Sep 12, 2011 at 11:58:09PM +0200, Hilko Bengen
wrote:
> ---
>  src/febootstrap_debian.ml |   65
+++++++++++++++++++++++++++++++++++++-------
>  1 files changed, 54 insertions(+), 11 deletions(-)
> 
Update on this one since I've been busy investigating a serious
bug in RHEL.

We decided that this functionality was great, but that it should be
available optionally, and across at least the Debian and Yum backends.
It would be useful for Fedora builds too.

----

On a separate note, febootstrap-supermin-helper could use a 9pfs
backend.  This would make boot times even faster, while reducing the
complexity of the boot process -- in other words, it's all upside.

It seems that the hard bit of this is modifying the 9p server inside
qemu to make it more flexible.  It would be good if the 9p server
could be driven a text configuration file:

  /etc directory
  /etc/passwd replace /tmp/base/etc/passwd mode 0644 owner root group root
  /usr/bin directory
  /usr/bin/ls host
  /usr/bin/vi host

This describes to qemu how to synthesize a filesystem.  Some files
comes from host files directly (hostfiles), and some files are
replacement files (from base.img).

Then all the febootstrap-supermin-helper backend would need to do
would be to unpack base.img and construct this simple text config
file.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines.  Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://et.redhat.com/~rjones/virt-top