Ovirt devel - Aug 2009 - [PATCH ovirt-node-image] fixes for edit-livecd

Patch set fixes issues with image size increase when using edit-livecd
Also address issue with ext4 root fs

Old edit-livecd just kept adding to the ext2 fs image which meant
it never decreased in size even if files were removed.

New functionality does multiple passes of ext2fs creation.  First pass
creates a new ext2fs from the image file inside the squashfs.  This
first pass is to allow the user to optionally increase the filesystem
size.  If the -s parameter is omitted the existing size is used and this
step is a nop.  The new ext2 fs is mounted to the ex-rw directory.

After ex-rw is modified by the user (either manually or through supplied
script) it is copied again to ex-min.  This step is done so that the
filesystem is minimized in size (i.e. file removals will actually reduce
the size of the ext2 image and therefore reduce the overall size of the
livecd)

NOTE: There seems to be an approximate 1MB increase in size even on a nop
edit-livecd.  This is not in the ext2 filesystem itself, it's seen in the
squashfs image.  Not sure why this happens.

Signed-off-by: Perry Myers <pmyers at redhat.com>
---
 edit-livecd |  123 ++++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 files changed, 110 insertions(+), 13 deletions(-)

diff --git a/edit-livecd b/edit-livecd
index d69ca9d..e94b1e6 100755
--- a/edit-livecd
+++ b/edit-livecd
@@ -31,7 +31,7 @@ CD=$NODEIMG_DEFAULT
 usage() {
     case $# in 1) warn "$1"; try_h; exit 1;; esac
     cat <<EOF
-Usage: $ME -i LiveCD.iso [-b bootparams] [-p program]
+Usage: $ME -i LiveCD.iso [-b bootparams] [-p program] [-s MB]
   -b BOOTPARAMS  optional parameters appended to the kernel command line
   -i LIVECD.iso  LiveCD ISO to edit (default: $NODEIMG_DEFAULT)
   -o OUTPUT.iso  specify the output file (required)
@@ -42,6 +42,7 @@ Usage: $ME -i LiveCD.iso [-b bootparams] [-p program]
                    the user (in another terminal) to modify the filesystem
                    manually.  Type <enter> when done, and the script
                    re-packages the ISO.
+  -s             Size of ext2 filesystem in MB (default is existing size)
   -h             display this help and exit

 EXAMPLES
@@ -69,14 +70,16 @@ set -e

 CODE OUTPUT_FILE+SIZE
 err=0 help=0
-while getopts :b:hi:o:p: c; do
+while getopts :b:hi:o:p:s: c; do
     case $c in
         i) CD=$OPTARG;;
         b) PARAMS=$OPTARG;;
         o) OUTPUT_FILE=$OPTARG;;
         p) CODE=$OPTARG;;
+        s) SIZE=$OPTARG;;
         h) help=1;;
         '?') err=1; warn "invalid option: \`-$OPTARG'";;
         :) err=1; warn "missing argument to \`-$OPTARG' option";;
@@ -86,9 +89,9 @@ done
 test $err = 1 && { try_h; exit 1; }
 test $help = 1 && { usage; exit 0; }

-# Require "-o OUTPUT_FILE"
+# Require "-o OUTPUT.iso"
 test -z "$OUTPUT_FILE" \
-  && { warn "no output file specified; use -o FILE.iso";
try_h; exit 1; }
+  && { warn "no output file specified; use -o OUTPUT.iso";
try_h; exit 1; }

 # Fail if there are any extra command-line arguments.
 if test $OPTIND -le $#; then
@@ -101,6 +104,8 @@ if [ $( id -u ) -ne 0 ]; then
     die "Must run as root"
 fi

+test selinuxenabled || { echo "selinux not enabled, aborting" ; exit
1 ; }
+
 # Check for some prerequisites.
 # "type" prints "PROG not found" if it's not in $PATH.
 type mkisofs
@@ -137,6 +142,71 @@ mnt() {
     addExit "df | grep $mp > /dev/null 2>&1 && umount -v
$mp"
 }

+# Create new ext2 filesystem and copy contents of one into the other
+# Size of ext2 filesystem is same as original size unless size parameter
+# is passed in
+create_ext2fs() {
+    local input_file=$1
+    local input_dir=$2
+    local output_file=$3
+    local output_dir=$4
+    local size_mb=$5
+
+    local size+    if [[ -n "$size_mb" ]]; then
+        size=$(( $size_mb * 1024 ))
+    fi
+
+    echo ">>> Mounting old ext3fs"
+    mnt "-t ext2 $input_file -o ro,loop" $input_dir
+    ext_used=$(df $WDIR/$input_dir | tail -1 | awk '{print $3}')
+    ext_max=$(df $WDIR/$input_dir | tail -1 | awk '{print $2}')
+
+    echo ">>> $input_file max_size=$ext_max
current_size=$ext_used"
+
+    local new_size+    if [[ -z "$size" ]]; then
+        new_size=$ext_max
+    elif [ $size -lt $ext_used ]; then
+        echo "New size of $size is less than the current used size
$ext_used"
+        exit 1
+    else
+	new_size=$size
+    fi
+
+    echo ">>> Size selected for new $output_file: $new_size"
+
+    echo ">>> Creating new ext3fs"
+    dd if=/dev/zero of=$output_file bs=1024 count=0 seek=${new_size}
+    mkfs.ext2 -b 4096 -m 1 -L "ovirt-node-image" -F $output_file
+    tune2fs -i 0 -c 0 -Odir_index -ouser_xattr,acl $output_file
+
+    echo ">>> Mounting new ext3fs"
+    mnt "-t ext2 $output_file -o rw,loop" $output_dir
+
+    SELINUX_ENFORCING=$(getenforce)
+    case $SELINUX_ENFORCING in
+        Enforcing) setenforce Permissive ;;
+        Permissive) ;;
+        *) echo "Do not run $0 with selinux in disabled mode" ; exit
1 ;;
+    esac
+
+    echo ">>> Copying data from old ext3fs to new ext3fs"
+    rsync -aX $WDIR/$input_dir/ $WDIR/$output_dir/
+
+    echo -e ">>> $(df $WDIR/$input_dir | tail -1)"
+    echo -e ">>> $(df $WDIR/$output_dir | tail -1)"
+    umount $WDIR/$output_dir
+    umount $WDIR/$input_dir
+
+    if [ "$SELINUX_ENFORCING" = "Enforcing" ]; then
+        setenforce Enforcing
+    fi
+
+    e2fsck -f -y $output_file || :
+    e2fsck -f -y $output_file || :
+}
+
 addExit "rm -rf $WDIR"

 ID_FS_LABEL= # initialize, in case vol_id fails
@@ -150,47 +220,73 @@ mnt "-t iso9660 $CD -o loop,ro" cd
 mnt "-t squashfs $WDIR/cd/LiveOS/squashfs.img -o ro,loop" sq

 # create writable copy of the new filesystem for the CD
-cp -pr $WDIR/cd $WDIR/cd-w
+rsync -a $WDIR/cd/ $WDIR/cd-w/

 # create writable copy of the filesystem for the new compressed
 # squashfs filesystem
-cp -pr $WDIR/sq $WDIR/sq-w
+rsync -a $WDIR/sq/ $WDIR/sq-w/
+echo ">>> Old LiveCD Size: $(du -b $CD)"
+echo ">>> Old Squash Size: $(du -b
$WDIR/cd/LiveOS/squashfs.img)"
+echo ">>> Old ext3fs Size: $(du -b
$WDIR/sq-w/LiveOS/ext3fs.img)"

-# mount root filesystem
-mnt "-t ext2 $WDIR/sq-w/LiveOS/ext3fs.img -o rw,loop" ex
+# Create copy of original ext2fs in case user wants to make
+# the file system larger than the original
+create_ext2fs $WDIR/sq-w/LiveOS/ext3fs.img ex-ro $WDIR/ext3fs-rw.img ex-rw
$SIZE
+
+echo ">>> New ext3fs Size: $(du -b $WDIR/ext3fs-rw.img)"
+
+echo ">>> Mounting new ext3fs"
+mnt "-t ext2 $WDIR/ext3fs-rw.img -o rw,loop" ex-rw

 echo ">>> Updating CD content"
 if [ -n "$CODE" ]; then
     (
-      cd $WDIR/ex
+      cd $WDIR/ex-rw
       set +e
       eval "$CODE"
       set -e
     )
 else
     echo "***"
-    echo "*** Pausing to allow manual changes.  Press any key to
continue."
+    echo "*** Pausing to allow manual changes."
+    echo "*** Make changes in $WDIR/ex-rw"
+    echo "*** Press any key to continue."
     echo "***"
     read
 fi

+# cleanup temporary and cache locations in the chroot, rhbz#508043
+rm -rf $WDIR/ex-rw/var/cache/yum
+rm -rf $WDIR/ex-rw/tmp
+mkdir $WDIR/ex-rw/tmp
+
 # Try to unmount.  But this is likely to fail, so let the user retry,
-# e.g., if he forgot to "cd" out of $WDIR/ex.
+# e.g., if he forgot to "cd" out of $WDIR/ex-rw.
 while :; do
   echo ">>> Unmounting ext3fs"
-  umount $WDIR/ex && break
+  umount $WDIR/ex-rw && break
   echo ">>> Unmounting the working file system copy failed"
   echo "***"
-  echo "*** Did you forget to 'cd' out of $WDIR/ex?"
+  echo "*** Did you forget to 'cd' out of $WDIR/ex-rw?"
   echo "***"
   echo "*** Press any key to repeat the attempt."
   echo "***"
   read
 done

+# Copy the filesystem again to make sure we minimize the image
+create_ext2fs $WDIR/ext3fs-rw.img ex-rw $WDIR/ext3fs-min.img ex-min $SIZE
+
+echo ">>> Min ext3fs Size: $(du -b $WDIR/ext3fs-min.img)"
+
+# Move the minimized ext2 filesystem into the squash directory
+mv -v $WDIR/ext3fs-min.img $WDIR/sq-w/LiveOS/ext3fs.img
+
 echo ">>> Compressing filesystem"
 mksquashfs $WDIR/sq-w/ $WDIR/cd-w/LiveOS/squashfs.img -noappend

+echo ">>> New Squash Size: $(du -b
$WDIR/cd-w/LiveOS/squashfs.img)"
+
 echo ">>> Recomputing MD5 sums"
 ( cd $WDIR/cd-w && find . -type f -not -name md5sum.txt \
     -not -path '*/isolinux/*' -print0 | xargs -0 -- md5sum >
md5sum.txt )
@@ -215,6 +311,7 @@ mkisofs \

 echo ">>> Implanting ISO MD5 Sum"
 implantisomd5 --force "$OUTPUT_FILE"
+echo ">>> New LiveCD Size: $(du -b $OUTPUT_FILE)"

 # The trap ... callbacks will unmount everything.
 set +e
-- 
1.6.2.5

---
 edit-livecd |    9 ++++++---
 1 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/edit-livecd b/edit-livecd
index e94b1e6..f519d5e 100755
--- a/edit-livecd
+++ b/edit-livecd
@@ -159,10 +159,13 @@ create_ext2fs() {

     echo ">>> Mounting old ext3fs"
     mnt "-t ext2 $input_file -o ro,loop" $input_dir
-    ext_used=$(df $WDIR/$input_dir | tail -1 | awk '{print $3}')
-    ext_max=$(df $WDIR/$input_dir | tail -1 | awk '{print $2}')
+    ext_type=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $2}')
+    ext_max=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $3}')
+    ext_used=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $4}')
+    ext_avail=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $5}')

-    echo ">>> $input_file max_size=$ext_max
current_size=$ext_used"
+    echo ">>> extfs image=$input_file"
+    echo ">>> type=$ext_type max_size=$ext_max
current_size=$ext_used available=$ext_avail"

     local new_size     if [[ -z "$size" ]]; then
-- 
1.6.2.5

---
 edit-livecd |   68 ++++++++++++++++++++++++++++++++++------------------------
 1 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/edit-livecd b/edit-livecd
index f519d5e..483daa1 100755
--- a/edit-livecd
+++ b/edit-livecd
@@ -142,10 +142,10 @@ mnt() {
     addExit "df | grep $mp > /dev/null 2>&1 && umount -v
$mp"
 }

-# Create new ext2 filesystem and copy contents of one into the other
-# Size of ext2 filesystem is same as original size unless size parameter
+# Create new filesystem and copy contents of one into the other
+# Size of filesystem is same as original size unless size parameter
 # is passed in
-create_ext2fs() {
+create_fs() {
     local input_file=$1
     local input_dir=$2
     local output_file=$3
@@ -157,14 +157,14 @@ create_ext2fs() {
         size=$(( $size_mb * 1024 ))
     fi

-    echo ">>> Mounting old ext3fs"
-    mnt "-t ext2 $input_file -o ro,loop" $input_dir
+    echo ">>> Mounting old fs image"
+    mnt "-t $fs_type $input_file -o ro,loop" $input_dir
     ext_type=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $2}')
     ext_max=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $3}')
     ext_used=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $4}')
     ext_avail=$(df -P -T $WDIR/$input_dir | tail -1 | awk '{print $5}')

-    echo ">>> extfs image=$input_file"
+    echo ">>> old ext3fs image=$input_file info:"
     echo ">>> type=$ext_type max_size=$ext_max
current_size=$ext_used available=$ext_avail"

     local new_size@@ -179,13 +179,13 @@ create_ext2fs() {

     echo ">>> Size selected for new $output_file: $new_size"

-    echo ">>> Creating new ext3fs"
+    echo ">>> Creating new fs"
     dd if=/dev/zero of=$output_file bs=1024 count=0 seek=${new_size}
-    mkfs.ext2 -b 4096 -m 1 -L "ovirt-node-image" -F $output_file
+    mkfs -t $fs_type -b 4096 -m 0 -L "ovirt-node-image" -F
$output_file
     tune2fs -i 0 -c 0 -Odir_index -ouser_xattr,acl $output_file

-    echo ">>> Mounting new ext3fs"
-    mnt "-t ext2 $output_file -o rw,loop" $output_dir
+    echo ">>> Mounting new fs"
+    mnt "-t $fs_type $output_file -o rw,loop" $output_dir

     SELINUX_ENFORCING=$(getenforce)
     case $SELINUX_ENFORCING in
@@ -194,20 +194,21 @@ create_ext2fs() {
         *) echo "Do not run $0 with selinux in disabled mode" ; exit
1 ;;
     esac

-    echo ">>> Copying data from old ext3fs to new ext3fs"
+    echo ">>> Copying data from old fs to new fs"
     rsync -aX $WDIR/$input_dir/ $WDIR/$output_dir/

-    echo -e ">>> $(df $WDIR/$input_dir | tail -1)"
-    echo -e ">>> $(df $WDIR/$output_dir | tail -1)"
-    umount $WDIR/$output_dir
+    echo -e ">>> OLD fs image: $(df -T $WDIR/$input_dir | tail
-1)"
+    echo -e ">>> NEW fs Image: $(df -T $WDIR/$output_dir | tail
-1)"
+    
+	umount $WDIR/$output_dir
     umount $WDIR/$input_dir

     if [ "$SELINUX_ENFORCING" = "Enforcing" ]; then
         setenforce Enforcing
     fi

-    e2fsck -f -y $output_file || :
-    e2fsck -f -y $output_file || :
+	echo ">>> Running fsck on new fs"
+    fsck -t $fs_type -f -y $output_file || :
 }

 addExit "rm -rf $WDIR"
@@ -228,18 +229,24 @@ rsync -a $WDIR/cd/ $WDIR/cd-w/
 # create writable copy of the filesystem for the new compressed
 # squashfs filesystem
 rsync -a $WDIR/sq/ $WDIR/sq-w/
-echo ">>> Old LiveCD Size: $(du -b $CD)"
-echo ">>> Old Squash Size: $(du -b
$WDIR/cd/LiveOS/squashfs.img)"
-echo ">>> Old ext3fs Size: $(du -b
$WDIR/sq-w/LiveOS/ext3fs.img)"

-# Create copy of original ext2fs in case user wants to make
-# the file system larger than the original
-create_ext2fs $WDIR/sq-w/LiveOS/ext3fs.img ex-ro $WDIR/ext3fs-rw.img ex-rw
$SIZE
+# get info
+cd_size=$(du -b $CD)
+sq_size=$(du -b $WDIR/cd/LiveOS/squashfs.img)
+fs_size=$(du -b $WDIR/sq-w/LiveOS/ext3fs.img)
+fs_type=$(file $WDIR/sq-w/LiveOS/ext3fs.img | awk '{print $5}')
+
+echo ">>> LiveCD iso Size: $cd_size"
+echo ">>> Squash image Size: $sq_size"
+echo ">>> fs image Size: $fs_size"
+echo ">>> fs type: $fs_type"

-echo ">>> New ext3fs Size: $(du -b $WDIR/ext3fs-rw.img)"
+# Create copy of original fs image in case user wants to make
+# the file system larger than the original
+create_fs $WDIR/sq-w/LiveOS/ext3fs.img ex-ro $WDIR/ext3fs-rw.img ex-rw $SIZE

-echo ">>> Mounting new ext3fs"
-mnt "-t ext2 $WDIR/ext3fs-rw.img -o rw,loop" ex-rw
+echo ">>> remounting new fs"
+mnt "-t $fs_type $WDIR/ext3fs-rw.img -o rw,loop" ex-rw

 echo ">>> Updating CD content"
 if [ -n "$CODE" ]; then
@@ -277,12 +284,15 @@ while :; do
   read
 done

+echo ">>> New fs image size: $(du -b $WDIR/ext3fs-rw.img)"
+
 # Copy the filesystem again to make sure we minimize the image
-create_ext2fs $WDIR/ext3fs-rw.img ex-rw $WDIR/ext3fs-min.img ex-min $SIZE
+echo ">>> Copying the filesystem again to minimize the
image"
+create_fs $WDIR/ext3fs-rw.img ex-rw $WDIR/ext3fs-min.img ex-min $SIZE

-echo ">>> Min ext3fs Size: $(du -b $WDIR/ext3fs-min.img)"
+echo ">>> Min fs image Size: $(du -b $WDIR/ext3fs-min.img)"

-# Move the minimized ext2 filesystem into the squash directory
+# Move the minimized filesystem into the squash directory
 mv -v $WDIR/ext3fs-min.img $WDIR/sq-w/LiveOS/ext3fs.img

 echo ">>> Compressing filesystem"
@@ -314,6 +324,8 @@ mkisofs \

 echo ">>> Implanting ISO MD5 Sum"
 implantisomd5 --force "$OUTPUT_FILE"
+
+echo ">>> Old LiveCD Size: $cd_size"
 echo ">>> New LiveCD Size: $(du -b $OUTPUT_FILE)"

 # The trap ... callbacks will unmount everything.
-- 
1.6.2.5