This set of patches introduces support for multipath devices for storage. Comments and suggestions are appreciated. Mike
Includes choosing device to install to ovirt-node-image stuff fix autoinstall to translate ovirt_init param Add manual device option Signed-off-by: Mike Burns <mburns at redhat.com> --- recipe/common-install.ks | 4 + recipe/common-post.ks | 11 +++ scripts/ovirt-config-storage | 171 ++++++++++++++++++++++++++++++++++++++---- scripts/ovirt-functions | 6 +- 4 files changed, 174 insertions(+), 18 deletions(-) diff --git a/recipe/common-install.ks b/recipe/common-install.ks index f3443b6..5c1e239 100644 --- a/recipe/common-install.ks +++ b/recipe/common-install.ks @@ -18,3 +18,7 @@ device scsi_wait_scan # multipath kmods device dm-multipath device dm-round-robin +device dm-emc +device dm-rdac +device dm-hp-sw +device scsi_dh_rdac diff --git a/recipe/common-post.ks b/recipe/common-post.ks index 0f09581..293c341 100644 --- a/recipe/common-post.ks +++ b/recipe/common-post.ks @@ -152,6 +152,7 @@ rm -f /etc/cron.daily/logrotate touch /var/lib/random-seed mkdir /live mkdir /boot +mkdir -p /var/cache/multipathd sed -i '/^files \/etc*/ s/^/#/' /etc/rwtab cat > /etc/rwtab.d/ovirt <<EOF dirs /var/lib/multipath @@ -162,7 +163,17 @@ files /var/cache/hald files /var/empty/sshd/etc/localtime files /var/lib/dbus files /var/lib/libvirt +files /var/lib/multipath +files /var/cache/multipathd empty /mnt empty /live empty /boot EOF + + +#use all hard-coded defaults for multipath +cat /dev/mull > /etc/multipath.conf + +#lvm.conf should use /dev/mapper and /dev/sdX devices +# and not /dev/dm-X devices +sed -i 's/preferred_names = \[ \]/preferred_names = [ "^\/dev\/mapper", "^\/dev\/[hsv]d" ]/g' /etc/lvm/lvm.conf diff --git a/scripts/ovirt-config-storage b/scripts/ovirt-config-storage index be22ef6..c3715fb 100755 --- a/scripts/ovirt-config-storage +++ b/scripts/ovirt-config-storage @@ -34,6 +34,68 @@ logging_min_size=5 data_min_size=5 swap_min_size=5 +# return sd name for given #:# identifier +get_sd_name() { + local id=$1 + local device_var=$2 + for device in $(ls /sys/block) + do + if [[ $id = $(cat /sys/block/$device/dev) ]]; then + eval $device_var=$device + return + fi + done + eval $device_var=1 +} + +# gets the dependent block devices for multipath devices +get_multipath_devices() { + local mpath_device=mpath-$(basename $1) + local deplist_var=$2 + local deplist="" + + #get dependencies for multipath device + local deps=$(dmsetup deps -u $mpath_device \ + | sed -r 's/\(([0-9]+), ([0-9]+)\)/\1:\2/g' \ + | sed 's/ /\n/g' | grep [0-9]:[0-9] ) + + local device="" + for dep in $deps + do + local device="" + get_sd_name $dep device + if [[ ! $device = 1 ]]; then + if [[ $deplist = "" ]]; then + deplist=$device + else + deplist="$deplist $device" + fi + fi + done + + eval $deplist_var='$deplist' +} + +#get Multipath device for given /dev/sdX device +#return sdX device if no multipath device +translate_multipath_device() { + #trim so that only sdX is stored, but support passing /dev/sdX + local dev=$1 + local mpath_var=$2 + + local basedev=$(basename $dev) + + local mpath_device=$(multipath -ll $dev |grep -n . | \ + grep "^1:" |awk '{print $1}' | sed 's/^1:/\/dev\/mapper\//g') + + if [ -z "$mpath_device" ]; then + mpath_device=$dev + fi + + eval $mpath_var=$mpath_device +} + + get_drive_size() { local drive=$1 @@ -53,8 +115,12 @@ get_drive_size() fi if [ -z "$udi" ]; then # If hal didn't find the device, it could be a virtio block device + # or a multipath device # In this case, use sfdisk -s to get the size - size=$(sfdisk -s $drive) + size=$(sfdisk -s $drive 2>/dev/null) + if [ -z "$size" ]; then + size=0 + fi size=$(echo "scale=0; $size * 1024" | bc -l) else size=$(hal-get-property --udi "$udi" --key storage.size) @@ -69,7 +135,7 @@ get_drive_size() size=$(echo "scale=0; $size / (1024 * 1024)" | bc -l) echo "$drive ($size MB)" - echo "Disk Identifier: $(basename "$udi")" + test -z "$udi" || echo "Disk Identifier: $(basename "$udi")" if [ -n "$space_var" ]; then eval $space_var=$size fi @@ -151,6 +217,29 @@ check_partition_sizes() return $rc } +manual_input() +{ + local manual_device + local return_var=$1 + while true; do + read -rp "Enter disk device path: " manual_device + if [ -z "$device" ]; then + return 1 + fi + translate_multipath_device $manual_device manual_device + eval $return_var="$manual_device" + if [ -n "$manual_device" ]; then + if [ -b "$(readlink -f $device)" ]; then + return 0 + fi + else + echo "Aborting." + return 1 + fi + + done +} + # Find a usable/selected storage device. # If there are none, give a diagnostic and return nonzero. # If there is just one, e.g., /dev/sda, treat it as selected (see below). @@ -161,7 +250,7 @@ check_partition_sizes() get_dev_name() { local udi_list=$(hal-find-by-capability --capability storage) - local byid_list=$(find /dev/disk/by-id -mindepth 1 -not -name '*-part*') + local byid_list=$(find /dev/disk/by-id -mindepth 1 -not -name '*-part*' 2>/dev/null) if test -z "$udi_list" -a -z "$byid_list"; then warn "ERROR: no usable storage devices detected" return 1 @@ -193,7 +282,33 @@ get_dev_name() # FIXME: workaround for detecting virtio block devices devices="$devices $(ls /dev/vd? 2> /dev/null | xargs)" - devices=$(echo $devices | tr ' ' '\n' | sort -u | xargs) + + # FIXME: workaround for detecting cciss devices + for dev in $(ls /dev/cciss 2>/dev/null); do + if [[ ! $dev =~ p[0-9]+\$ ]]; then + devices="$devices /dev/cciss/dev" + fi + done + + # Include mulitpath devices + local devs_to_remove + for dev in $(dmsetup ls --target=multipath | awk '{print $1}'); do + devices="$devices /dev/mapper/$dev" + local sd_devs="" + get_multipath_devices $dev sd_devs + devs_to_remove="${devs_to_remove} ${sd_devs}" + done + + # Remove /dev/sd* devices that are part of a multipath device + local dev_list + for dev in $devices + do + if [[ ! "$devs_to_remove" =~ "$(basename $dev)" ]]; then + dev_list="$dev_list $dev" + fi + done + + devices=$(echo $dev_list | tr ' ' '\n' | sort -u | xargs) local num_devices=$(echo "$devices" | wc -w) # If there's only one device, use it. @@ -209,9 +324,10 @@ get_dev_name() get_drive_size $d >&2 done local choices="$devices Abort" - select device in $choices + select device in $choices "Manual Selection" do test "$device" = Abort && return 1 + test "$device" = "Manual selection" && manual_input device test -z "$device" && continue echo "$device" return 0 @@ -375,6 +491,17 @@ wipe_lvm_on_disk() done } + +reread_partitions() +{ + local drive=$1 + if [[ $drive =~ "^/dev/mapper" ]]; then + kpartx -a -p p $drive + else + blockdev --rereadpt $drive + fi +} + perform_partitioning() { if [[ -z "$HOSTVGDRIVE" && "$OVIRT_ISCSI_ENABLED" != "y" ]]; then @@ -404,15 +531,20 @@ perform_partitioning() log "Partitioning drive: $BOOTDRIVE" log "Wiping old boot sector" dd if=/dev/zero of=$BOOTDRIVE bs=1024K count=1 - blockdev --rereadpt $BOOTDRIVE + reread_partitions $BOOTDRIVE partprobe -s $BOOTDRIVE log "Creating boot partition" parted $BOOTDRIVE -s "mklabel ${LABEL_TYPE}" parted $BOOTDRIVE -s "mkpartfs primary ext2 0M ${boot_size_si}M" + reread_partitions $BOOTDRIVE + partboot=$BOOTDRIVE1 + if [ ! -e $partboot ]; then + partboot=${BOOTDRIVE}p1 + fi # sleep to ensure filesystems are created before continuing sleep 10 - mke2fs ${BOOTDRIVE}1 -L Boot - tune2fs -c 0 -i 0 ${BOOTDRIVE}1 + mke2fs ${partboot} -L Boot + tune2fs -c 0 -i 0 ${partboot} log "Completed!" return fi @@ -422,7 +554,7 @@ perform_partitioning() # FIXME: save a backup copy, just in case? log "Wiping old boot sector" dd if=/dev/zero of=$ROOTDRIVE bs=1024K count=1 - blockdev --rereadpt $ROOTDRIVE + reread_partitions $ROOTDRIVE partprobe -s $ROOTDRIVE log "Labeling Drive: $ROOTDRIVE" parted $ROOTDRIVE -s "mklabel ${LABEL_TYPE}" @@ -435,12 +567,19 @@ perform_partitioning() let RootBackup_end=${ROOT_SIZE}*2 parted $ROOTDRIVE -s "mkpart primary ext2 0M ${ROOT_SIZE}M" parted $ROOTDRIVE -s "mkpart primary ext2 ${ROOT_SIZE}M ${RootBackup_end}M" + reread_partitions $ROOTDRIVE + partroot=${ROOTDRIVE}1 + partrootbackup=${ROOTDRIVE}2 + if [ ! -e $partroot ]; then + partroot=${ROOTDRIVE}p1 + partrootbackup=${ROOTDRIVE}p2 + fi # sleep to ensure filesystems are created before continuing sleep 10 - mke2fs ${ROOTDRIVE}1 -L Root - mke2fs ${ROOTDRIVE}2 -L RootBackup - tune2fs -c 0 -i 0 ${ROOTDRIVE}1 - tune2fs -c 0 -i 0 ${ROOTDRIVE}2 + mke2fs ${partroot} -L Root + mke2fs ${partrootbackup} -L RootBackup + tune2fs -c 0 -i 0 ${partroot} + tune2fs -c 0 -i 0 ${partrootbackup} log "Creating LVM partition" if [ $ROOTDRIVE == $HOSTVGDRIVE ]; then @@ -454,6 +593,7 @@ perform_partitioning() parted $HOSTVGDRIVE -s "set $hostvgpart lvm on" parted $ROOTDRIVE -s "print" udevadm settle 2> /dev/null || udevsettle + reread_partitions $HOSTVGDRIVE # sync GPT to the legacy MBR partitions if [ "gpt" == "$LABEL_TYPE" ]; then @@ -702,8 +842,9 @@ DATA_SIZE=${OVIRT_VOL_DATA_SIZE:-$default_data_size} if [ -n "$OVIRT_INIT" ]; then # if present, use the drive selected with 'ovirt_init' boot parameter # setting these the same until kernel cmdline argument implemented - ROOTDRIVE=$OVIRT_INIT - HOSTVGDRIVE=$OVIRT_INIT + translate_multipath_device $OVIRT_INIT DRIVE + ROOTDRIVE=$DRIVE + HOSTVGDRIVE=$DRIVE get_drive_size $ROOTDRIVE ROOTDRIVESPACE fi diff --git a/scripts/ovirt-functions b/scripts/ovirt-functions index ff2b016..4027613 100644 --- a/scripts/ovirt-functions +++ b/scripts/ovirt-functions @@ -332,7 +332,7 @@ unmount_logging_services() { cd /etc/init.d for prg in $(lsof -Fc +D /var/log|grep ^c|sort -u); do srv=$(grep -l ${prg#c}$ *) - service $srv stop + service $srv stop 6>&- 7>&- services="$services $srv" done # debugging help @@ -364,7 +364,7 @@ mount_logging() { rmdir $log2 restorecon -rv /var/log for srv in $services; do - service $srv start + service $srv start 6>&- 7>&- done return 0 @@ -392,7 +392,7 @@ unmount_logging() { return $rc fi for srv in $services; do - service $srv start + service $srv start 6>&- 7>&- done return 0 -- 1.6.6.1
Mike Burns
2010-Feb-24 17:47 UTC
[Ovirt-devel] [PATCH node 2/5] Check for negative data partition with default storage config
If the data partition is a negative size, don't offer default sizes. Instead force the user to enter partition sizes Signed-off-by: Mike Burns <mburns at redhat.com> --- scripts/ovirt-config-storage | 28 ++++++++++++++++++---------- 1 files changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/ovirt-config-storage b/scripts/ovirt-config-storage index c3715fb..a16126c 100755 --- a/scripts/ovirt-config-storage +++ b/scripts/ovirt-config-storage @@ -17,7 +17,7 @@ trap 'exit $?' 1 2 13 15 # check that we're not booted from local storage; if so then exit with an error if is_booted_from_local_disk; then - die "You cannot configure storage on a running system. Please boot from CD/USB to configure local storage." + die "You cannot configure storage on a running system. Please boot from CD/USB to configure storage." fi default_overcommit=0.5 @@ -361,9 +361,12 @@ do_configure() printf "* For the Data partition, a size of -1 indicates that the\n" printf " partition should use up the remaining space on the disk.\n\n" - do_review - if ask_yes_or_no "Use these default values ([Y]es/[N]o)?"; then - return + if do_review; then + if ask_yes_or_no "Use these default values ([Y]es/[N]o)?"; then + return + fi + else + printf "Selected disk is too small for default storage configuration" fi local space_left=$HOSTVGDRIVESPACE @@ -438,8 +441,9 @@ do_review() { if [[ -z "$ROOTDRIVE" && "$OVIRT_ISCSI_ENABLED" != "y" ]]; then printf "\nNo Root storage device selected.\n" - return + return 1 fi + local is_negative=0 if [ "$OVIRT_ISCSI_ENABLED" != "y" ]; then local data_size_display="$DATA_SIZE MB" @@ -448,15 +452,17 @@ do_review() if [ $ROOTDRIVE == $HOSTVGDRIVE ]; then local remaining_mb=$(( $ROOTDRIVESPACE - $SWAP_SIZE \ - $ROOT_SIZE * 2 - $CONFIG_SIZE - $LOGGING_SIZE )) + test $remaining_mb -lt 0 && is_negative=1 else local remaining_mb=$(( $HOSTVGDRIVESPACE - $SWAP_SIZE - $CONFIG_SIZE - $LOGGING_SIZE )) + test $remaining_mb -lt 0 && is_negative=1 fi data_size_display="$remaining_mb MB" fi cat <<EOF -The local disk will be repartitioned as follows: +The selected disk will be repartitioned as follows: =============================================== Root Drive: $(get_drive_size $ROOTDRIVE) HostVG Drive: $(get_drive_size $HOSTVGDRIVE) @@ -470,7 +476,7 @@ EOF else cat <<EOF -The local disk will be repartitioned as follows: +The selected disk will be repartitioned as follows: =============================================== Boot Drive: $(get_drive_size $BOOTDRIVE) Boot partition size: $BOOT_SIZE @@ -478,6 +484,8 @@ The local disk will be repartitioned as follows: EOF fi + + return ${is_negative-0} } # cleanup lvms on selected disk @@ -693,9 +701,9 @@ do_confirm() "$w8" \ "$wb$sp$w" \ "$wb$sp$w" \ - "$wb If you proceed, this will destroy all data on $w" \ - "$wb your local system, and your hard disk will be $w" \ - "$wb irreversably reconfigured $w" \ + "$wb If you proceed, all data on your selected storage $w" \ + "$wb device will be destroyed and your hard disk $w" \ + "$wb will be irreversably reconfigured $w" \ "$wb$sp$w" \ "$wb$sp$w" \ "$w8" \ -- 1.6.6.1
Mike Burns
2010-Feb-24 17:47 UTC
[Ovirt-devel] [PATCH node 3/5] Handle existing HostVGs on the storage
Allow the user to install on a device that already contains HostVG. Fail install if HostVG exists on a different drive. Signed-off-by: Mike Burns <mburns at redhat.com> --- scripts/ovirt-config-storage | 81 +++++++++++++++++++++++++++++++++++++---- 1 files changed, 73 insertions(+), 8 deletions(-) diff --git a/scripts/ovirt-config-storage b/scripts/ovirt-config-storage index a16126c..44389f3 100755 --- a/scripts/ovirt-config-storage +++ b/scripts/ovirt-config-storage @@ -349,6 +349,24 @@ do_configure() printf "\n\nPlease select the disk to use for the HostVG.\n\n" HOSTVGDRIVE=$(get_dev_name) || return 0 + local skipped=false + if check_existing_hostvg $HOSTVGDRIVE devs; then + for dev in $devs + do + printf "Removing HostVG on $dev will erase the drive and cannot be undone\n" + if ask_yes_or_no "Do you want to remove HostVG from $dev (y/n)?"; then + start_log + if ! wipe_lvm_on_disk $dev; then + stop_log + return 1 + fi + stop_log + else + skipped=true + fi + done + fi + $skipped && printf "Installation cannot proceed with existing HostVG.\n" && return 0 get_drive_size $HOSTVGDRIVE HOSTVGDRIVESPACE echo $HOSTVGDRIVESPACE fi @@ -488,15 +506,59 @@ EOF return ${is_negative-0} } +#Check for an existing HostVG on any device on the system. +# Return 0 if then is a HostVG found, unless only one found is on $1 +# Return 1 if no HostVG found or only found on $1 +check_existing_hostvg() +{ + local install_dev=$1 + local devices_var=$2 + if [ -z "$install_dev" ]; then + devices="$(pvs -o pv_name,vg_name --noheadings | \ + grep "HostVG" | awk '{print $1}' )" + else + devices="$(pvs -o pv_name,vg_name --noheadings | \ + grep -v ${install_dev} | grep "HostVG" | awk '{print $1}' )" + fi + rc=1 + if [ -n "$devices" ]; then + printf "\n" + printf "There appears to already be an installation on another device:\n" + for device in $devices; do + get_multipath_devices ${device%p[0-9]} sd_dev + sd_dev=$(echo $sd_dev | awk '{print $1}') + udi=$(hal-find-by-property --key block.device --string /dev/${sd_dev}) + printf "\t$device ($(basename "$udi"))\n" + done + printf "The installation cannot proceed until the device is removed\n" + printf "from the system of the HostVG volume group is removed.\n" + rc=0 + fi + + test -z $devices_var || eval $devices_var=$devices + + return $rc + +} + # cleanup lvms on selected disk # - remove mounted filesystems # - remove LVM volumes and groups wipe_lvm_on_disk() { + local dev=${1-$HOSTVGDRIVE} unmount_logging for vg in $(pvs -o vg_name --noheadings $HOSTVGDRIVE* 2>/dev/null|sort -u); do + if pvs -o pv_name,vg_name --noheadings | \ + grep $vg | grep -v -q $dev* 2>/dev/null; then + log "The volume group \"$vg\" spans multiple disks." + log "This operation cannot complete. Please manullay" + log "cleanup the storage using standard linux tools." + return 1 + fi wipe_volume_group $vg done + return 0 } @@ -867,19 +929,22 @@ if [ "$1" == "AUTO" ]; then log "Beginning automatic disk partitioning.\n" if [ -n "$OVIRT_INIT" ]; then # do not format if HostVG exists on selected disk... - pvs -o vg_name --noheadings $HOSTVGDRIVE* 2>/dev/null|grep -q -m1 "HostVG" - existingHostVG=$? + check_existing_hostvg $HOSTVGDRIVE + existingHostVG=$? # ... unless overridden by ovirt_firstboot parameter - if is_firstboot || [ $existingHostVG -ne 0 ]; then + if is_firstboot || [ $existingHostVG -ne 0 ]; then if check_partition_sizes; then - log "Partitioning hard disk..." - perform_partitioning + log "Partitioning hard disk..." + perform_partitioning + exit $? fi - else + else log "Skip disk partitioning, HostVG exists" - fi + exit 1 + fi else - log "Missing device parameter: unable to partition any disk" + log "Missing device parameter: unable to partition any disk" + exit 2 fi else OPTIONS="\"Enable iSCSI Target\" \"Configure Storage\" \"Review\" \"Commit Changes And Quit\" \"Return To Menu\"" -- 1.6.6.1
Mike Burns
2010-Feb-24 17:47 UTC
[Ovirt-devel] [PATCH node 4/5] update uninstall steps to handle multipath
Signed-off-by: Mike Burns <mburns at redhat.com> --- scripts/ovirt-config-uninstall | 51 +++++++++++++++++++++++++++------------ 1 files changed, 35 insertions(+), 16 deletions(-) diff --git a/scripts/ovirt-config-uninstall b/scripts/ovirt-config-uninstall index 820eead..a2c9b3c 100755 --- a/scripts/ovirt-config-uninstall +++ b/scripts/ovirt-config-uninstall @@ -29,24 +29,43 @@ EOF if ask_yes_or_no "Do you wish to continue and uninstall this node ([Y]es/[N]o)?"; then if [ -d /dev/HostVG ]; then - log "Uninstalling node" - log "Detaching logging" - unmount_logging - log "Removing volume group" - wipe_volume_group "HostVG" - partition=$(findfs LABEL=Root) - if [ -n "$partition" ]; then - log "Removing partitions" - drive=$(echo $partition | awk '{ print substr($0, 1, length($0) - 1) }') - parted -s $drive "rm 1" - parted -s $drive "rm 2" + log "Uninstalling node" + log "Detaching logging" + start_log + # multipathd holds all mounts under /var in a private namespace + service multipathd stop 6>&- 7>&- + rm -f /var/lib/multipath/bindings + unmount_logging + log "Removing volume group" + wipe_volume_group "HostVG" + partition=$(readlink -f $(findfs LABEL=Root)) + if [ -n "$partition" ]; then + log "Removing partitions" + eval $(echo $partition | awk ' { + print "drive=" substr($0,1,length($1)-1); + print "drive2=" substr($0,1,length($1)-2); + }') + if [ ! -e "$drive" ]; then + drive="$drive2" + partpv="$drive}p2" + else + partpv="${drive}2" + fi + parted -s $drive "rm 1" + pvremove ${partpv} + parted -s $drive "rm 2" parted -s $drive "rm 3" - fi - printf "Finished uninstalling node." + fi + #restart multipath + multipath -F + multipath -v3 + service multipathd start 6>&- 7>&- + log "Finished uninstalling node." + stop_log else - log "There is no installed node instance to remove." - log "Aborting" - exit 1 + log "There is no installed node instance to remove." + log "Aborting" + exit 1 fi else log "Aborted" -- 1.6.6.1
Mike Burns
2010-Feb-24 17:47 UTC
[Ovirt-devel] [PATCH node 5/5] Check for plymouth running before trying to kill it
Don't try to kill plymouth if it's already dead Signed-off-by: Mike Burns <mburns at redhat.com> --- scripts/ovirt-functions | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ovirt-functions b/scripts/ovirt-functions index 4027613..1db89ae 100644 --- a/scripts/ovirt-functions +++ b/scripts/ovirt-functions @@ -382,8 +382,9 @@ unmount_logging() { log "Unmounting log partition" # plymouthd keeps /var/log/boot.log - plymouth --quit - plymouth --wait + if plymouth --ping ; then + plymouth --quit + fi unmount_logging_services umount /var/log -- 1.6.6.1
Reasonably Related Threads
- [PATCH node] Handle space in storage wwid
- RESEND: [PATCH node 1/3] enables ability for a common shared root
- [PATCH node] iscsi remote root basework This lays most of the groundwork for iscsi installation and configuration. At this time configuring iscsi is disabled due to multiple issues with dependent pieces.
- [PATCH node 1/3] Enables ability to have a common shared root
- [PATCH][node REPOST] Improve performance of multipath translations