Ovirt devel - Aug 2008 - [PATCH] use YUM cache instead of pungi in build-all.sh

appliance-creator doesn't need install tree with anaconda images so we can
use YUM cache to avoid re-downloading RPMs for successive image rebuilds

Signed-off-by: Alan Pevec <apevec at redhat.com>
---
 .gitignore                            |    1 +
 build-all.sh                          |  178 ++++++++++++++-------------------
 ovirt-host-creator/Makefile           |    2 +-
 ovirt-host-creator/ovirt-cd           |    6 +-
 wui-appliance/create-wui-appliance.sh |   21 +++--
 5 files changed, 94 insertions(+), 114 deletions(-)

diff --git a/.gitignore b/.gitignore
index 72271a2..d36c16e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 wui/src/log/*
 wui/src/tmp/*
 wui/src/db/schema.rb
+tmp/
diff --git a/build-all.sh b/build-all.sh
index 3d7d5ee..451c6cf 100755
--- a/build-all.sh
+++ b/build-all.sh
@@ -2,12 +2,11 @@
 
 #
 # build all oVirt components
-#
-# - create local YUM repository with Fedora subset required by oVirt
+# - create oVirt host image (livecd-creator)
 # - create local YUM repository with ovirt-wui and ovirt-host-image-pxe RPMs
-# - create oVirt admin appliance
+# - create oVirt admin appliance (appliance-creator)
 
-# Requires: createrepo httpd kvm libvirt livecd-tools pungi
+# Requires: createrepo kvm libvirt livecd-tools appliance-tools
 
 PATH=$PATH:/sbin:/usr/sbin
 
@@ -20,22 +19,22 @@ cd $(dirname $0)
 BASE=$(pwd)
 F_REL=9
 ARCH=$(uname -i)
-HTDOCS=/var/www/html
-OVIRT=$HTDOCS/ovirt
-PUNGI=$HTDOCS/pungi
-PUNGIKS=$PUNGI/pungi.ks
-DEP_RPMS="createrepo httpd kvm libvirt livecd-tools pungi
appliance-tools"
+NODE=$BASE/ovirt-host-creator
+WUI=$BASE/wui-appliance
+BUILD=$BASE/tmp
+OVIRT=$BUILD/ovirt
+CACHE=$BUILD/cache
+DEP_RPMS="createrepo kvm libvirt livecd-tools appliance-tools"
 
 usage() {
     case $# in 1) warn "$1"; try_h; exit 1;; esac
     cat <<EOF
-Usage: $ME [-w] [-n] [-p] [-s] [-a] [-c] [-v git|release|none] [-e eth]
+Usage: $ME [-w] [-n] [-s] [-a] [-c] [-v git|release|none] [-e eth]
   -w: update oVirt WUI RPMs
   -n: update oVirt Managed Node RPMs
-  -p: update pungi repository
   -s: include SRPMs and produce source ISO
   -a: updates all (WUI, Node, Appliance)
-  -c: cleanup local repos (pungi and ovirt)
+  -c: cleanup local oVirt repo and YUM cache
   -v: update version type (git, release, none) default is git
   -e: ethernet device to use as bridge (i.e. eth1)
   -h: display this help and exit
@@ -53,17 +52,16 @@ bumpver() {
 }
 
 update_wui=0 update_node=0
-update_pungi=0 update_app=0
+update_app=0
 include_src=0
 cleanup=0
 version_type=git
 bridge err=0 help=0
-while getopts wnpsahcv:e: c; do
+while getopts wnsacv:e:h c; do
     case $c in
         w) update_wui=1;;
         n) update_node=1;;
-        p) update_pungi=1;;
         s) include_src=1;;
         a) update_wui=1; update_node=1; update_app=1;;
         c) cleanup=1;;
@@ -77,8 +75,6 @@ while getopts wnpsahcv:e: c; do
 done
 test $err = 1 && { try_h; exit 1; }
 test $help = 1 && { usage; exit 0; }
-test $include_src = 1 -a "$update_pungi" = 0 &&
-  usage "Need to specify -p when including source"
 test "$version_type" != "git" -a "$version_type"
!= "release" \
     -a "$version_type" != "none" \
     && usage "version type must be git, release or none"
@@ -96,32 +92,18 @@ if [ $? -ne 0 ]; then
     die "Must have $DEP_RPMS installed"
 fi
 set -e
-echo -n "pungi-1.2.18.1-1 or newer "
-$BASE/ovirt-host-creator/rpm-compare.py GE 0 pungi 1.2.18.1 1
-echo ok
 echo -n "appliance-tools-002-1 or newer "
-$BASE/ovirt-host-creator/rpm-compare.py GE 0 appliance-tools 002 1
+$NODE/rpm-compare.py GE 0 appliance-tools 002 1
 echo ok
 set +e
 
-mkdir -p $PUNGI
 mkdir -p $OVIRT
+mkdir -p $CACHE
 
-# cleanup repository folders
+# cleanup repository and YUM cache
 if [ $cleanup = 1 ]; then
-    update_pungi=1
-    rm -rf $PUNGI/*
     rm -rf $OVIRT/*
-fi
-
-# If doing either a node or app build, make sure http is running
-if [ $update_app = 1 -o $update_node = 1 -o $update_pungi != 0 ]; then
-    lokkit --service http
-    service httpd status > /dev/null 2>&1 ||
-      service httpd start > /dev/null 2>&1
-    service libvirtd status > /dev/null 2>&1 ||
-      service libvirtd start > /dev/null 2>&1
-    service libvirtd reload
+    rm -rf $CACHE/*
 fi
 
 # stop execution on any error
@@ -141,65 +123,69 @@ if [ $update_wui = 1 ]; then
     createrepo .
 fi
 
-# build Fedora subset required for oVirt
-if [ $update_pungi != 0 ]; then
-    pungi_flags="-GC"
-
-    fedora_mirror=http://mirrors.fedoraproject.org/mirrorlist
-    # use Fedora + updates
-    currentbadupdates=''
-    cat > $PUNGIKS << EOF
+fedora_mirror=http://mirrors.fedoraproject.org/mirrorlist
+# use Fedora + updates
+currentbadupdates=''
+cat > $NODE/repos.ks << EOF
 repo --name=f$F_REL \
   --mirrorlist=$fedora_mirror?repo=fedora-$F_REL&arch=\$basearch
 repo --name=f$F_REL-updates \
   --mirrorlist=$fedora_mirror?repo=updates-released-f$F_REL&arch=\$basearch
$currentbadupdates
 EOF
-    # + ovirt.org repo for updates not yet in Fedora
-    # + local ovirt repo with locally rebuilt ovirt* RPMs ( options -w and -n )
-    #   if not available, ovirt* RPMs from ovirt.org will be used
-    excludepkgs-    if [[ -f $OVIRT/repodata/repomd.xml ]]; then
-        excludepkgs='--excludepkgs=ovirt*'
-        cat >> $PUNGIKS << EOF
-repo --name=ovirt --baseurl=http://localhost/ovirt
+# + ovirt.org repo for updates not yet in Fedora
+# + local ovirt repo with locally rebuilt ovirt* RPMs ( options -w and -n )
+#   if not available, ovirt* RPMs from ovirt.org will be used
+excludepkgs+if [[ -f $OVIRT/repodata/repomd.xml ]]; then
+    excludepkgs='--excludepkgs=ovirt*'
+    cat >> $NODE/repos.ks << EOF
+repo --name=ovirt --baseurl=file://$OVIRT
 EOF
-    fi
-    cat >> $PUNGIKS << EOF
+fi
+cat >> $NODE/repos.ks << EOF
 repo --name=ovirt-org \
   --baseurl=http://ovirt.org/repos/ovirt/$F_REL/\$basearch $excludepkgs
 EOF
-    if [ $include_src != 0 ]; then
-        cat >> $PUNGIKS << EOF
-repo --name=f$F_REL-src \
-  --mirrorlist=$fedora_mirror?repo=fedora-source-$F_REL&arch=\$basearch
-repo --name=f$F_REL-updates-src \
- 
--mirrorlist=$fedora_mirror?repo=updates-released-source-f$F_REL&arch=\$basearch
$currentbadupdates
-repo --name=ovirt-org-src \
-  --baseurl=http://ovirt.org/repos/ovirt/$F_REL/src $excludepkgs
+if [ $include_src != 0 ]; then
+    cat > $OVIRT/sources.repo << EOF
+[main]
+cachedir=$CACHE
+keepcache=1
+[f$F_REL-src]
+name=f$F_REL-src
+mirrorlist=$fedora_mirror?repo=fedora-source-$F_REL&arch=\$basearch
+enabled=1
+gpgcheck=0
+[f$F_REL-updates-src]
+name=f$F_REL-updates-src
+mirrorlist=$fedora_mirror?repo=updates-released-source-f$F_REL&arch=\$basearch
$currentbadupdates
+enabled=1
+gpgcheck=0
+[ovirt-org-src]
+name=ovirt-org-src
+baseurl=http://ovirt.org/repos/ovirt/$F_REL/src
+exclude=${excludepkgs#--excludepkgs=}
+enabled=1
+gpgcheck=0
 EOF
-    else
-        pungi_flags+=" --nosource"
-    fi
-
-    cd $BASE
-    cat >> $PUNGIKS << EOF
-
-%packages
+if [ -n "$excludepkgs" ]; then
+    cat >> $OVIRT/sources.repo << EOF
+[ovirt-src]
+name=ovirt-src
+baseurl=file://$OVIRT
+enabled=1
+gpgcheck=0
 EOF
-    # merge package lists from all oVirt kickstarts
-    # exclude ovirt-host-image* (chicken-egg: built at the next step
-    # using repo created here)
-    egrep -hv "^-|^ovirt-host-image" \
-        ovirt-host-creator/common-pkgs.ks \
-        wui-appliance/common-pkgs.ks \
-        | sort -u >> $PUNGIKS
-    cd $PUNGI
-    pungi --ver=$F_REL $pungi_flags -c $PUNGIKS --force
-    if [ $include_src != 0 ]; then
-        pungi --ver=$F_REL -I  --sourceisos --nosplitmedia -c $PUNGIKS --force
-    fi
-    restorecon -r .
 fi
+        mkdir -p $BUILD/source
+        cd $BUILD/source
+        # FIXME yumdownloader doesn't resolve @groups
+        # XXX global yum repos are included
+        yumdownloader -c $OVIRT/sources.repo --source --resolve \
+          $(egrep -hv "^-|^ovirt-host-image" \
+            $NODE/common-pkgs.ks \
+            $WUI/common-pkgs.ks | sort -u)
+    fi
 
 # build oVirt host image; note that we unconditionally rebuild the
 # ovirt-managed-node RPM, since it is now needed for the managed node
@@ -214,15 +200,10 @@ if [ $update_node = 1 ]; then
     cd $OVIRT
     createrepo .
 
-    cd $BASE/ovirt-host-creator
+    cd $NODE
     rm -rf rpm-build
-    cat > repos.ks << EOF
-repo --name=f$F_REL --baseurl=http://localhost/pungi/$F_REL/$ARCH/os
-repo --name=ovirt --baseurl=http://localhost/ovirt
-
-EOF
     bumpver
-    make rpms
+    make rpms YUMCACHE=$CACHE
     rm -f $OVIRT/ovirt-host-image*rpm
     cp rpm-build/ovirt-host-image*rpm $OVIRT
     cd $OVIRT
@@ -231,22 +212,9 @@ fi
 
 # build oVirt admin appliance
 if [ $update_app == 1 ]; then
-    cd $BASE/wui-appliance
+    cd $WUI
     make clean
-    cat > repos.ks << EOF
-url --url http://localhost/pungi/$F_REL/$ARCH/os
-EOF
-    excludepkgs-    if [[ -f $OVIRT/repodata/repomd.xml ]]; then
-        excludepkgs='--excludepkgs=ovirt*'
-        cat >> repos.ks << EOF
-repo --name=ovirt --baseurl=http://localhost/ovirt
-EOF
-    fi
-    cat >> repos.ks << EOF
-repo --name=ovirt-org --baseurl=http://ovirt.org/repos/ovirt/$F_REL/$ARCH
$excludepkgs
-
-EOF
+    cp $NODE/repos.ks $WUI/repos.ks
     make
 
     bridge_flag@@ -254,7 +222,7 @@ EOF
         bridge_flag="-e $bridge"
     fi
 
-    ./create-wui-appliance.sh \
+    ./create-wui-appliance.sh -y $CACHE \
         -k wui-rel.ks \
         $bridge_flag
 
diff --git a/ovirt-host-creator/Makefile b/ovirt-host-creator/Makefile
index 1f0122f..a912e84 100644
--- a/ovirt-host-creator/Makefile
+++ b/ovirt-host-creator/Makefile
@@ -14,7 +14,7 @@ repos.ks: repos.ks.in
 
 build: ovirt.ks common-install.ks common-pkgs.ks common-post.ks repos.ks
 	./rpm-compare.py GE 0 livecd-tools 017.1 1
-	./ovirt-cd
+	./ovirt-cd $(YUMCACHE)
 
 tar: clean build
 	mv $$(cat iso-file) ovirt.iso
diff --git a/ovirt-host-creator/ovirt-cd b/ovirt-host-creator/ovirt-cd
index 5678f14..9a2ba8f 100755
--- a/ovirt-host-creator/ovirt-cd
+++ b/ovirt-host-creator/ovirt-cd
@@ -20,8 +20,12 @@
 PATH=/sbin:/bin:/usr/bin
 
 KICKSTART=ovirt.ks
+CACHE+if [ -n "$1" ]; then
+  CACHE=--cache=$1
+fi
 
 LABEL=ovirt-`date +%Y%m%d%H%M`
-livecd-creator --skip-minimize -c $KICKSTART -f $LABEL 1>&2 &&
+livecd-creator $CACHE --skip-minimize -c $KICKSTART -f $LABEL 1>&2
&&
 
 echo $LABEL.iso > iso-file
diff --git a/wui-appliance/create-wui-appliance.sh
b/wui-appliance/create-wui-appliance.sh
index b15cc76..1683237 100755
--- a/wui-appliance/create-wui-appliance.sh
+++ b/wui-appliance/create-wui-appliance.sh
@@ -18,9 +18,10 @@ imgdir=$IMGDIR_DEFAULT
 usage() {
     case $# in 1) warn "$1"; try_h; exit 1;; esac
     cat <<EOF
-Usage: $ME [-c] [-d image_dir] [-k kickstart] [-e eth]
-  -d: directory to place virtual disk (default: $IMGDIR_DEFAULT)
+Usage: $ME [-c] [-d image_dir] [-y yumcachedir] [-k kickstart] [-e eth]
   -c: compress the image (qcow2 compressed)
+  -d: directory to place virtual disk (default: $IMGDIR_DEFAULT)
+  -y: YUM cache directory to use
   -k: appliance kickstart file
   -e: ethernet device to use as bridge (i.e. eth1)
   -h: display this help and exit
@@ -31,10 +32,12 @@ err=0 help=0
 compress=0
 bridge kickstart-while getopts :d:k:he: c; do
+yumcache+while getopts :cd:y:k:e:h c; do
     case $c in
-        d) imgdir=$OPTARG;;
         c) compress=1;;
+        d) imgdir=$OPTARG;;
+        y) yumcache=--cache=$OPTARG;;
         k) kickstart=$OPTARG;;
         e) bridge=$OPTARG;;
         h) help=1;;
@@ -43,6 +46,9 @@ while getopts :d:k:he: c; do
         *) err=1; warn "internal error: \`-$OPTARG' not
handled";;
     esac
 done
+if [ -n "$1" ]; then
+  name=$1
+fi
 test $err = 1 && { try_h; exit 1; }
 test $help = 1 && { usage; exit 0; }
 
@@ -275,15 +281,16 @@ virsh undefine $NAME > /dev/null 2>&1
 if [ -n "$kickstart" ]; then
     mkdir -p tmp
     set -e
-    appliance-creator --config $kickstart --name $NAME --tmpdir $(pwd)/tmp
+    appliance-creator --config $kickstart --name $NAME \
+      --tmpdir=$(pwd)/tmp $yumcache
     # FIXME add --compress option to appliance-creator
     if [ $compress -ne 0 ]; then
         printf "Compressing the image..."
         qemu-img convert -c $NAME-sda.raw -O qcow2 "$imgdir/$IMGNAME"
-        rm ovirt-appliance-sda.raw
+        rm $NAME-sda.raw
     else
         printf "Moving the image..."
-        mv ovirt-appliance-sda.raw "$imgdir/$IMGNAME"
+        mv $NAME-sda.raw "$imgdir/$IMGNAME"
         restorecon -v "$imgdir/$IMGNAME"
     fi
     echo done
-- 
1.5.5.1

On Wed, 2008-08-13 at 02:58 +0200, Alan Pevec wrote:
> appliance-creator doesn't need install tree with anaconda images so we
can
> use YUM cache to avoid re-downloading RPMs for successive image rebuilds
> 
> Signed-off-by: Alan Pevec <apevec at redhat.com>
ACK.  Tried this, on second install of appliance went from taking almost
an hour down to about 10 minutes.

-j

On Wed, Aug 13, 2008 at 02:58:04AM +0200, Alan Pevec
wrote:
> appliance-creator doesn't need install tree with anaconda images so we
can
> use YUM cache to avoid re-downloading RPMs for successive image rebuilds
> 
> Signed-off-by: Alan Pevec <apevec at redhat.com>
> ---
>  .gitignore                            |    1 +
>  build-all.sh                          |  178
++++++++++++++-------------------
>  ovirt-host-creator/Makefile           |    2 +-
>  ovirt-host-creator/ovirt-cd           |    6 +-
>  wui-appliance/create-wui-appliance.sh |   21 +++--
>  5 files changed, 94 insertions(+), 114 deletions(-)
> 
Well done, ACK. Echo what j said re: time reduction.

Good day,
Steve