Libguestfs - Jul 2017 - [PATCH 0/5] various improvements for make-template.mk

Hi,

this series improves the generation of Debian templates, automating
basically almost all the steps needed.
Also, it fixes the generation of aarch64 images with NVRAM, which is
currently removed too soon, and thus not saved and compressed.

Thanks,


Pino Toscano (5):
  builder: templates: update Debian preseed from Stretch
  builder: templates: generalize debian location
  builder: templates: improve debian.preseed
  builder: templates: save NVRAM before removing guest
  builder: templates: pass empty proxy to d-i when not set

 builder/templates/debian.preseed   | 80 ++++++++++++++++++++++++++++++--------
 builder/templates/make-template.ml | 59 +++++++++++++++++-----------
 2 files changed, 99 insertions(+), 40 deletions(-)

-- 
2.9.4

Update our debian.preseed file from a Stretch example file, leaving all
the customization done already.
---
 builder/templates/debian.preseed | 76 ++++++++++++++++++++++++++++++++--------
 1 file changed, 61 insertions(+), 15 deletions(-)

diff --git a/builder/templates/debian.preseed b/builder/templates/debian.preseed
index 54d4253..22bf460 100644
--- a/builder/templates/debian.preseed
+++ b/builder/templates/debian.preseed
@@ -1,4 +1,4 @@
-#### Contents of the preconfiguration file (for wheezy)
+#### Contents of the preconfiguration file (for stretch)
 ### Localization
 # Preseeding only locale sets language, country and locale.
 d-i debian-installer/locale string en_US
@@ -11,8 +11,7 @@ d-i debian-installer/locale string en_US
 #d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8
 
 # Keyboard selection.
-# keymap is an alias for keyboard-configuration/xkb-keymap
-d-i keymap select us
+d-i keyboard-configuration/xkb-keymap select us
 # d-i keyboard-configuration/toggle select No toggling
 
 ### Network configuration
@@ -30,7 +29,7 @@ d-i netcfg/choose_interface select auto
 
 # To set a different link detection timeout (default is 3 seconds).
 # Values are interpreted as seconds.
-#d-i netcfg/link_detection_timeout string 10
+#d-i netcfg/link_wait_timeout string 10
 
 # If you have a slow dhcp server and the installer times out waiting for
 # it, this might be useful.
@@ -116,8 +115,8 @@ d-i passwd/make-user boolean false
 # Root password, either in clear text
 d-i passwd/root-password password builder
 d-i passwd/root-password-again password builder
-# or encrypted using an MD5 hash.
-#d-i passwd/root-password-crypted password [MD5 hash]
+# or encrypted using a crypt(3)  hash.
+#d-i passwd/root-password-crypted password [crypt(3) hash]
 
 # To create a normal user account.
 #d-i passwd/user-fullname string Debian User
@@ -125,8 +124,8 @@ d-i passwd/root-password-again password builder
 # Normal user's password, either in clear text
 #d-i passwd/user-password password insecure
 #d-i passwd/user-password-again password insecure
-# or encrypted using an MD5 hash.
-#d-i passwd/user-password-crypted password [MD5 hash]
+# or encrypted using a crypt(3) hash.
+#d-i passwd/user-password-crypted password [crypt(3) hash]
 # Create the first user with the specified UID instead of the default.
 #d-i passwd/user-uid string 1010
 
@@ -155,8 +154,8 @@ d-i clock-setup/ntp boolean true
 
 # Alternatively, you may specify a disk to partition. If the system has only
 # one disk the installer will default to using that, but otherwise the device
-# name must be given in traditional, non-devfs format (so e.g. /dev/hda or
-# /dev/sda, and not e.g. /dev/discs/disc0/disc).
+# name must be given in traditional, non-devfs format (so e.g. /dev/sda
+# and not e.g. /dev/discs/disc0/disc).
 # For example, to use the first SCSI/SATA hard disk:
 #d-i partman-auto/disk string /dev/sda
 # In addition, you'll need to specify the method to use.
@@ -179,7 +178,7 @@ d-i partman-lvm/confirm_nooverwrite boolean true
 # You can choose one of the three predefined partitioning recipes:
 # - atomic: all files in one partition
 # - home:   separate /home partition
-# - multi:  separate /home, /usr, /var, and /tmp partitions
+# - multi:  separate /home, /var, and /tmp partitions
 d-i partman-auto/choose_recipe select atomic
 
 # Or provide a recipe of your own...
@@ -220,6 +219,9 @@ d-i partman/choose_partition select finish
 d-i partman/confirm boolean true
 d-i partman/confirm_nooverwrite boolean true
 
+# When disk encryption is enabled, skip wiping the partitions beforehand.
+#d-i partman-auto-crypto/erase_disks boolean false
+
 ## Partitioning using RAID
 # The method should be set to "raid".
 #d-i partman-auto/method string raid
@@ -314,11 +316,12 @@ d-i partman/confirm_nooverwrite boolean true
 # authentication. Warning: Insecure, not recommended.
 #d-i debian-installer/allow_unauthenticated boolean true
 
+# Uncomment this to add multiarch configuration for i386
+#d-i apt-setup/multiarch string i386
+
+
 ### Package selection
-#tasksel tasksel/first multiselect standard, web-server
-# If the desktop task is selected, install the kde and xfce desktops
-# instead of the default gnome desktop.
-#tasksel tasksel/desktop multiselect kde, xfce
+#tasksel tasksel/first multiselect standard, web-server, kde-desktop
 
 # Individual additional packages to install
 #d-i pkgsel/include string openssh-server build-essential
@@ -332,6 +335,48 @@ d-i partman/confirm_nooverwrite boolean true
 # popular and include it on CDs.
 popularity-contest popularity-contest/participate boolean false
 
+### Boot loader installation
+# Grub is the default boot loader (for x86). If you want lilo installed
+# instead, uncomment this:
+#d-i grub-installer/skip boolean true
+# To also skip installing lilo, and install no bootloader, uncomment this
+# too:
+#d-i lilo-installer/skip boolean true
+
+
+# This is fairly safe to set, it makes grub install automatically to the MBR
+# if no other operating system is detected on the machine.
+d-i grub-installer/only_debian boolean true
+
+# This one makes grub-installer install to the MBR if it also finds some other
+# OS, which is less safe as it might not be able to boot that other OS.
+d-i grub-installer/with_other_os boolean true
+
+# Due notably to potential USB sticks, the location of the MBR can not be
+# determined safely in general, so this needs to be specified:
+#d-i grub-installer/bootdev  string /dev/sda
+# To install to the first device (assuming it is not a USB stick):
+#d-i grub-installer/bootdev  string default
+
+# Alternatively, if you want to install to a location other than the mbr,
+# uncomment and edit these lines:
+#d-i grub-installer/only_debian boolean false
+#d-i grub-installer/with_other_os boolean false
+#d-i grub-installer/bootdev  string (hd0,1)
+# To install grub to multiple disks:
+#d-i grub-installer/bootdev  string (hd0,1) (hd1,1) (hd2,1)
+
+# Optional password for grub, either in clear text
+#d-i grub-installer/password password r00tme
+#d-i grub-installer/password-again password r00tme
+# or encrypted using an MD5 hash, see grub-md5-crypt(8).
+#d-i grub-installer/password-crypted password [MD5 hash]
+
+# Use the following option to add additional boot parameters for the
+# installed system (if supported by the bootloader installer).
+# Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string nousb
+
 ### Finishing up the installation
 # During installations from serial console, the regular virtual consoles
 # (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next
@@ -383,3 +428,4 @@ d-i finish-install/reboot_in_progress note
 # directly, or use the apt-install and in-target commands to easily install
 # packages and run commands in the target system.
 #d-i preseed/late_command string apt-install zsh; in-target chsh -s /bin/zsh
+
-- 
2.9.4

Add an helper function to translate our architecture to the Debian
naming, and use it to get the location for all the architectures.
---
 builder/templates/make-template.ml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/builder/templates/make-template.ml
b/builder/templates/make-template.ml
index ab89488..9c46f79 100755
--- a/builder/templates/make-template.ml
+++ b/builder/templates/make-template.ml
@@ -349,6 +349,15 @@ and string_of_arch = function
   | PPC64le -> "ppc64le"
   | S390X -> "s390x"
 
+and debian_arch_of_arch = function
+  | X86_64 -> "amd64"
+  | Aarch64 -> "arm64"
+  | Armv7 -> "armhf"
+  | I686 -> "i386"
+  | PPC64 -> "ppc64"
+  | PPC64le -> "ppc64el"
+  | S390X -> "s390x"
+
 and filename_of_os os arch ext    match os with
   | Fedora ver ->
@@ -587,9 +596,9 @@ and make_location os arch        *)
      sprintf "http://mirror.centos.org/centos-7/%d/os/x86_64/" major
 
-  | Debian (_, dist), X86_64 ->
-     sprintf
"http://deb.debian.org/debian/dists/%s/main/installer-amd64"
-             dist
+  | Debian (_, dist), arch ->
+     sprintf
"http://deb.debian.org/debian/dists/%s/main/installer-%s"
+             dist (debian_arch_of_arch arch)
 
   | Fedora ver, Aarch64 ->
      sprintf
"https://download.fedoraproject.org/pub/fedora-secondary/releases/%d/Server/aarch64/os/"
ver
-- 
2.9.4

- set the mirror to deb.debian.org, which is the official redirector, so
  generated images will use the closest mirror depending on their
  location
- automake the task selection, using the choice we want
- do a full upgrade of the distro, so the template is already up-to-date
- install grub on the default device, which should also be the only one
  in the automated installation
---
 builder/templates/debian.preseed | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/builder/templates/debian.preseed b/builder/templates/debian.preseed
index 22bf460..b56eb67 100644
--- a/builder/templates/debian.preseed
+++ b/builder/templates/debian.preseed
@@ -96,7 +96,7 @@ d-i netcfg/wireless_wep string
 # If you select ftp, the mirror/country string does not need to be set.
 #d-i mirror/protocol string ftp
 d-i mirror/country string manual
-d-i mirror/http/hostname string ftp.uk.debian.org
+d-i mirror/http/hostname string deb.debian.org
 d-i mirror/http/directory string /debian
 #d-i mirror/http/proxy string
 
@@ -321,13 +321,13 @@ d-i partman/confirm_nooverwrite boolean true
 
 
 ### Package selection
-#tasksel tasksel/first multiselect standard, web-server, kde-desktop
+tasksel tasksel/first multiselect standard, ssh-server
 
 # Individual additional packages to install
 #d-i pkgsel/include string openssh-server build-essential
 # Whether to upgrade packages after debootstrap.
 # Allowed values: none, safe-upgrade, full-upgrade
-#d-i pkgsel/upgrade select none
+d-i pkgsel/upgrade select full-upgrade
 
 # Some versions of the installer can report back on what software you have
 # installed, and what software you use. The default is not to report back,
@@ -356,7 +356,7 @@ d-i grub-installer/with_other_os boolean true
 # determined safely in general, so this needs to be specified:
 #d-i grub-installer/bootdev  string /dev/sda
 # To install to the first device (assuming it is not a USB stick):
-#d-i grub-installer/bootdev  string default
+d-i grub-installer/bootdev string default
 
 # Alternatively, if you want to install to a location other than the mbr,
 # uncomment and edit these lines:
-- 
2.9.4

When installing guests that need NVRAM variables, the cleanup of the
guest with `virsh undefine` will remove that file too, which is not what
we want.  Instead, compress the NVRAM file right before the cleanup,
to ensure we have it.

Also, fix the filename for it, removing the double "-nvram" suffixes.
---
 builder/templates/make-template.ml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/builder/templates/make-template.ml
b/builder/templates/make-template.ml
index 9c46f79..164e146 100755
--- a/builder/templates/make-template.ml
+++ b/builder/templates/make-template.ml
@@ -97,6 +97,11 @@ let rec main ()    (* Choose a random temporary disk name. *)
   let tmpout = sprintf "%s.img" tmpname in
 
+  (* Create the final output name (actually not quite final because
+   * we will xz-compress it).
+   *)
+  let output = filename_of_os os arch "" in
+
   (* Some architectures need EFI boot. *)
   let tmpefivars      match os, arch with
@@ -146,6 +151,20 @@ let rec main ()    if pid = 0 then Unix.execvp
"virt-install" vi;
   let _, pstat = Unix.waitpid [] pid in
   check_process_status_for_errors pstat;
+  (* If there were NVRAM variables, move them to the final name and
+   * compress them.  Doing this operation later means the cleanup of
+   * the guest will remove them as well (because of --nvram).
+   *)
+  let nvram +    match tmpefivars with
+    | Some (_, vars) ->
+       let f = sprintf "%s-nvram" output in
+       let cmd = sprintf "mv %s %s" (quote vars) (quote f) in
+       if Sys.command cmd <> 0 then exit 1;
+       let cmd = sprintf "xz -f --best %s" (quote f) in
+       if Sys.command cmd <> 0 then exit 1;
+       Some (f ^ ".xz")
+    | None -> None in
   cleanup_libvirt_guest ();
   ignore (Sys.command "sync");
 
@@ -211,11 +230,6 @@ let rec main ()              (if is_selinux_os os then
" --selinux-relabel" else "") in
   if Sys.command cmd <> 0 then exit 1;
 
-  (* Create the final output name (actually not quite final because
-   * we will xz-compress it).
-   *)
-  let output = filename_of_os os arch "" in
-
   (* Sparsify and copy to output name. *)
   printf "Sparsifying ...\n%!";
   let cmd @@ -230,20 +244,6 @@ let rec main ()    if Sys.command cmd <> 0
then exit 1;
   let output = output ^ ".xz" in
 
-  (* If there were NVRAM variables, move them to the final name and
-   * compress them too.
-   *)
-  let nvram -    match tmpefivars with
-    | Some (_, vars) ->
-       let f = sprintf "%s-nvram" output in
-       let cmd = sprintf "mv %s %s-nvram" (quote vars) (quote f) in
-       if Sys.command cmd <> 0 then exit 1;
-       let cmd = sprintf "xz -f --best %s-nvram" (quote f) in
-       if Sys.command cmd <> 0 then exit 1;
-       Unix.unlink vars;
-       Some (f ^ ".xz")
-    | None -> None in
   printf "Template completed: %s\n%!" output;
 
   (* Construct the index fragment, but don't create this for the private
-- 
2.9.4

When there is no proxy set, pass an empty string to the Debian
Installer.  This way, the installer will not ask for a proxy.
---
 builder/templates/make-template.ml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/builder/templates/make-template.ml
b/builder/templates/make-template.ml
index 164e146..fe9a44f 100755
--- a/builder/templates/make-template.ml
+++ b/builder/templates/make-template.ml
@@ -726,7 +726,11 @@ and make_virt_install_command os arch ks tmpname tmpout
tmpefivars location
   let proxy      let p = try Some (Sys.getenv "http_proxy") with
Not_found -> None in
     match p with
-    | None -> ""
+    | None ->
+       (match os with
+       | Fedora _ | RHEL _ | CentOS _ | Ubuntu _ -> ""
+       | Debian _ -> "mirror/http/proxy="
+       )
     | Some p ->
        match os with
        | Fedora _ | RHEL _ | CentOS _ -> "proxy=" ^ p
-- 
2.9.4

On Mon, Jul 10, 2017 at 04:46:23PM +0200, Pino Toscano
wrote:
> When there is no proxy set, pass an empty string to the Debian
> Installer.  This way, the installer will not ask for a proxy.
> ---
>  builder/templates/make-template.ml | 6 +++++-
>  1 file changed, 5 insertions(+), 1 deletion(-)
> 
> diff --git a/builder/templates/make-template.ml
b/builder/templates/make-template.ml
> index 164e146..fe9a44f 100755
> --- a/builder/templates/make-template.ml
> +++ b/builder/templates/make-template.ml
> @@ -726,7 +726,11 @@ and make_virt_install_command os arch ks tmpname
tmpout tmpefivars location
>    let proxy >      let p = try Some (Sys.getenv "http_proxy")
with Not_found -> None in
>      match p with
> -    | None -> ""
> +    | None ->
> +       (match os with
> +       | Fedora _ | RHEL _ | CentOS _ | Ubuntu _ -> ""
> +       | Debian _ -> "mirror/http/proxy="
> +       )
>      | Some p ->
>         match os with
>         | Fedora _ | RHEL _ | CentOS _ -> "proxy=" ^ p
> -- 
> 2.9.4
ACK series, thanks.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v

Checking out sources from https://github.com/libguestfs/libguestfs ...

/var/tmp/tmppyH7Jl/libguestfs /var/tmp/tmppyH7Jl
Reset branch 'master'
Branch master set up to track remote branch master from origin.
Your branch is up-to-date with 'origin/master'.
>From github.com:libguestfs/libguestfs
   4cea914d2..28c375701  master     -> origin/master
Updating 4cea914d2..28c375701
Fast-forward
 builder/templates/debian.preseed   | 80 ++++++++++++++++++++++++++++++--------
 builder/templates/make-template.ml | 59 +++++++++++++++++-----------
 2 files changed, 99 insertions(+), 40 deletions(-)
/var/tmp/tmppyH7Jl

Applying patches ...

/var/tmp/tmppyH7Jl/libguestfs /var/tmp/tmppyH7Jl
error: patch failed: builder/templates/debian.preseed:1
error: builder/templates/debian.preseed: patch does not apply
Applying: builder: templates: update Debian preseed from Stretch
Patch failed at 0001 builder: templates: update Debian preseed from Stretch
The copy of the patch that failed is found in: .git/rebase-apply/patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am
--abort".