Libguestfs - May 2013 - remotely mounting client disks in p2v server

Just spent a little time poking around w/ p2v and am attaching the
following pseudo-code for consideration. Note the patches aren't commit
ready yet, just looking for thoughts

- The first is a fix to get client image building process working on F17.
  It seems the version of ksflatten there didn't expand the nested %includes
  which was causing errors (also --interpreter image-minimizer resulted in
errors)

- The next patch adds an option to the ui to allow the user to specify that
  they would like to mount a disk as opposed to copy it to the server. This
  option is currently mutually exclusive w/ the 'convert' checkbox.

  An option still needs to be added to the ui to toggle how to export the
  local disks (nbd, iscsi, etc). Would adding this as a global flag (eg applies
  to all disks) work?

- The next patch stubs out some code which to process the disks for which
'mount'
  was select. The actual calls to export the nbd/iscsi disk have yet to be added
  though the mount command is issued to the server.

- The final patch stubs out the corresponding server side functionality which
interprets
  the mount command and includes placeholders to connect to the remote uri
locally
  and pass the remote disks onto the converter.

The commands to setup the nbd/iscsi server and client (on the p2v client and
server respectively)
are simple enough though I still have to look into how to use the existing ssh
tunnel
for security.

Does this looks to be in about the right direction? Other thoughts?

  -Mo

---
 p2v/image-builder/common-minimizer.ks     | 2 +-
 p2v/image-builder/common-pkgs.ks          | 2 ++
 p2v/image-builder/common-post-nochroot.ks | 2 ++
 p2v/image-builder/common-post.ks          | 2 ++
 p2v/image-builder/p2v-post.ks             | 2 ++
 p2v/image-builder/virt-p2v-image.ks       | 6 ------
 6 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/p2v/image-builder/common-minimizer.ks
b/p2v/image-builder/common-minimizer.ks
index cd2c8d6..f1ba537 100644
--- a/p2v/image-builder/common-minimizer.ks
+++ b/p2v/image-builder/common-minimizer.ks
@@ -5,7 +5,7 @@
 #
 # Matthew Booth <mbooth at redhat.com> - 18/4/2011
 
-%post --nochroot --interpreter image-minimizer
+%post --nochroot
 # lokkit is just an install-time dependency; we can remove
 # it afterwards, which we do here
 droprpm system-config-*
diff --git a/p2v/image-builder/common-pkgs.ks b/p2v/image-builder/common-pkgs.ks
index 2dfd86e..1c6eaca 100644
--- a/p2v/image-builder/common-pkgs.ks
+++ b/p2v/image-builder/common-pkgs.ks
@@ -1,3 +1,4 @@
+%packages --excludedocs --nobase
 # Direct requirements
 rubygem-virt-p2v
 dejavu-sans-fonts
@@ -52,3 +53,4 @@ ql2200-firmware
 ql23xx-firmware
 ql2400-firmware
 ql2500-firmware
+%end
diff --git a/p2v/image-builder/common-post-nochroot.ks
b/p2v/image-builder/common-post-nochroot.ks
index 58015a2..fee305d 100644
--- a/p2v/image-builder/common-post-nochroot.ks
+++ b/p2v/image-builder/common-post-nochroot.ks
@@ -1,3 +1,4 @@
+%post --nochroot
 %include version.ks
 
 PRODUCT='Virt P2V'
@@ -58,3 +59,4 @@ cp $INSTALL_ROOT/etc/issue $INSTALL_ROOT/etc/issue.net
 if [ -f "$INSTALL_ROOT/initrd0.img" ]; then
   mv -v "$INSTALL_ROOT/initrd0.img"
"$LIVE_ROOT/isolinux/initrd0.img"
 fi
+%end
diff --git a/p2v/image-builder/common-post.ks b/p2v/image-builder/common-post.ks
index f591d67..268965d 100644
--- a/p2v/image-builder/common-post.ks
+++ b/p2v/image-builder/common-post.ks
@@ -1,3 +1,4 @@
+%post
 # -*-Shell-script-*-
 echo "Starting Kickstart Post"
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
@@ -34,3 +35,4 @@ sed -i '/rm -f \/etc\/yp.conf/d' $DHSCRIPT
 localedef --list-archive | grep -v -i -E 'en_US.utf8' |xargs localedef
--delete-from-archive
 mv /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl
 /usr/sbin/build-locale-archive
+%end
diff --git a/p2v/image-builder/p2v-post.ks b/p2v/image-builder/p2v-post.ks
index 31fa8e3..bf58692 100644
--- a/p2v/image-builder/p2v-post.ks
+++ b/p2v/image-builder/p2v-post.ks
@@ -1,3 +1,4 @@
+%post
 # Run virt-p2v
 if [ ! -e /etc/rc.d/rc.local ]; then
     echo "#!/bin/sh" > /etc/rc.d/rc.local
@@ -53,3 +54,4 @@ done
 done
 /sbin/poweroff
 EOF
+%end
diff --git a/p2v/image-builder/virt-p2v-image.ks
b/p2v/image-builder/virt-p2v-image.ks
index dca8840..4b40a85 100644
--- a/p2v/image-builder/virt-p2v-image.ks
+++ b/p2v/image-builder/virt-p2v-image.ks
@@ -2,19 +2,13 @@
 
 %include common-install.ks
 
-%packages --excludedocs --nobase
 %include common-pkgs.ks
-%end
 
-%post
 %include common-post.ks
 %include p2v-post.ks
-%end
 
 %include common-minimizer.ks
 
-%post --nochroot
 %include common-post-nochroot.ks
-%end
 
 %include common-manifest-post.ks
-- 
1.7.11.7

---
 p2v/client/lib/virt-p2v/ui/convert.rb | 43 +++++++++++++++++++++++------------
 p2v/client/lib/virt-p2v/ui/p2v.ui     | 14 ++++++++++++
 2 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/p2v/client/lib/virt-p2v/ui/convert.rb
b/p2v/client/lib/virt-p2v/ui/convert.rb
index 244125e..4fe61ab 100644
--- a/p2v/client/lib/virt-p2v/ui/convert.rb
+++ b/p2v/client/lib/virt-p2v/ui/convert.rb
@@ -14,6 +14,8 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+# TODO add option to toggle mounting disks using nbd or iscsi
+
 require 'gtk2'
 
 require 'virt-p2v/blockdevice'
@@ -26,9 +28,10 @@ module VirtP2V::UI::Convert
     CONVERT_NETWORK_DEVICE  = 1
 
     CONVERT_FIXED_CONVERT   = 0
-    CONVERT_FIXED_DEVICE    = 1
-    CONVERT_FIXED_PROGRESS  = 2
-    CONVERT_FIXED_SIZE_GB   = 3
+    CONVERT_FIXED_MOUNT     = 1
+    CONVERT_FIXED_DEVICE    = 2
+    CONVERT_FIXED_PROGRESS  = 3
+    CONVERT_FIXED_SIZE_GB   = 4
 
     CONVERT_REMOVABLE_CONVERT   = 0
     CONVERT_REMOVABLE_DEVICE    = 1
@@ -99,6 +102,7 @@ module VirtP2V::UI::Convert
         VirtP2V::FixedBlockDevice.all_devices.each { |dev|
             fixed = @fixeds.append
             fixed[CONVERT_FIXED_CONVERT]    = true
+            fixed[CONVERT_FIXED_MOUNT]      = false
             fixed[CONVERT_FIXED_DEVICE]     = dev.device
             fixed[CONVERT_FIXED_PROGRESS]   = 0
             fixed[CONVERT_FIXED_SIZE_GB]    = dev.size / 1024 / 1024 / 1024
@@ -134,6 +138,8 @@ module VirtP2V::UI::Convert
                             method(:update_values))
         ui.register_handler('convert_fixed_select_toggled',
                             method(:convert_fixed_select_toggled))
+        ui.register_handler('mount_fixed_select_toggled',
+                            method(:mount_fixed_select_toggled))
         ui.register_handler('convert_removable_select_toggled',
                             method(:convert_removable_select_toggled))
         ui.register_handler('convert_network_select_toggled',
@@ -319,16 +325,19 @@ module VirtP2V::UI::Convert
             raise InvalidUIState if memory.nil?
             @converter.memory = memory * 1024 * 1024
 
-            # Check that at least 1 fixed storage device is selected
-            fixed = false
+            # Check that at least 1 fixed or mounted storage device is selected
+            has_disk = false
             @converter.disks.clear
             @fixeds.each { |model, path, iter|
                 if iter[CONVERT_FIXED_CONVERT] then
-                    fixed = true
+                    has_disk = true
                     @converter.disks << iter[CONVERT_FIXED_DEVICE]
+                elsif iter[CONVERT_FIXED_MOUNT] then
+                    has_disk = true
+                    @converter.mounts << iter[CONVERT_FIXED_MOUNT]
                 end
             }
-            raise InvalidUIState unless fixed
+            raise InvalidUIState unless has_disk
 
             # Populate removables and nics, although these aren't required
to be
             # selected for the ui state to be valid
@@ -373,17 +382,16 @@ module VirtP2V::UI::Convert
         memory = Integer(memory) rescue nil
         return false if memory.nil?
 
-        # Check that at least 1 fixed storage device is selected
-        fixed = false
+        # Check that at least 1 fixed or mounted storage device is selected
+        has_disk = false
         @fixeds.each { |model, path, iter|
-            if iter[CONVERT_FIXED_CONVERT] then
-                fixed = true
+            if iter[CONVERT_FIXED_CONVERT] ||
+               iter[CONVERT_FIXED_MOUNT]   then
+                has_disk = true
                 break
             end
         }
-        return false unless fixed
-
-        return true
+        return has_disk
     end
 
     def self.convert_cpus_changed
@@ -416,6 +424,13 @@ module VirtP2V::UI::Convert
     def self.convert_fixed_select_toggled(widget, path)
         iter = @fixeds.get_iter(path)
         iter[CONVERT_FIXED_CONVERT] = !iter[CONVERT_FIXED_CONVERT]
+        iter[CONVERT_FIXED_MOUNT]   = false if iter[CONVERT_FIXED_CONVERT]
+    end
+
+    def self.mount_fixed_select_toggled(widget, path)
+        iter = @fixeds.get_iter(path)
+        iter[CONVERT_FIXED_MOUNT]   = !iter[CONVERT_FIXED_MOUNT]
+        iter[CONVERT_FIXED_CONVERT] = false if iter[CONVERT_FIXED_MOUNT]
     end
 
     def self.convert_removable_select_toggled(widget, path)
diff --git a/p2v/client/lib/virt-p2v/ui/p2v.ui
b/p2v/client/lib/virt-p2v/ui/p2v.ui
index a27bd20..f4d24a8 100644
--- a/p2v/client/lib/virt-p2v/ui/p2v.ui
+++ b/p2v/client/lib/virt-p2v/ui/p2v.ui
@@ -276,6 +276,20 @@
                                   </object>
                                 </child>
                                 <child>
+                                  <object
class="GtkTreeViewColumn" id="treeviewcolumn13">
+                                    <property
name="title">Mount</property>
+                                    <property
name="clickable">True</property>
+                                    <child>
+                                      <object
class="GtkCellRendererToggle" id="mount_fixed_select">
+                                        <signal name="toggled"
handler="mount_fixed_select_toggled" swapped="no"/>
+                                      </object>
+                                      <attributes>
+                                        <attribute
name="active">0</attribute>
+                                      </attributes>
+                                    </child>
+                                  </object>
+                                </child>
+                                <child>
                                   <object
class="GtkTreeViewColumn" id="treeviewcolumn5">
                                     <property
name="title">Device</property>
                                     <child>
-- 
1.7.11.7

---
 p2v/client/lib/virt-p2v/connection.rb | 12 ++++++++++++
 p2v/client/lib/virt-p2v/converter.rb  | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/p2v/client/lib/virt-p2v/connection.rb
b/p2v/client/lib/virt-p2v/connection.rb
index c3e5537..b923144 100644
--- a/p2v/client/lib/virt-p2v/connection.rb
+++ b/p2v/client/lib/virt-p2v/connection.rb
@@ -243,6 +243,18 @@ class Connection
         }
     end
 
+    def mount(uri, disk_type, &cb)
+        raise NotConnectedError if @channel.nil?
+
+        run(completion) {
+            @channel.write("MOUNT #{uri} #{disk_type}\n")
+            result = parse_return
+
+            Gtk.queue { cb.call(result) }
+        }
+    end
+
+
     private
 
     def run(cb)
diff --git a/p2v/client/lib/virt-p2v/converter.rb
b/p2v/client/lib/virt-p2v/converter.rb
index 7cb4e33..ef7541f 100644
--- a/p2v/client/lib/virt-p2v/converter.rb
+++ b/p2v/client/lib/virt-p2v/converter.rb
@@ -37,6 +37,9 @@ module VirtP2V
 #   path          Detected
 #   is_block      1
 #   format        raw
+# mounts        Editable, default to none
+#   device        Detected
+#   uri           Detected
 # removables    Editable, default to all
 #   device        Detected
 #   type          Detected
@@ -47,7 +50,7 @@ module VirtP2V
 
 class Converter
     attr_accessor :profile, :name, :cpus, :memory, :arch, :debug
-    attr_reader :features, :disks, :removables, :nics
+    attr_reader :features, :disks, :mounts, :removables, :nics
 
     attr_reader :connection
 
@@ -74,6 +77,13 @@ class Converter
                 }, cb)
             },
             lambda { |cb|
+                iterate(@mounts.map { |dev|
+                    lambda { |cb2|
+                      mount(dev, status, cb2)
+                    }
+                }, cb)
+            },
+            lambda { |cb|
                 if @debug
                     begin
                         @connection.options({ "DEBUG" => 1 },
&cb)
@@ -140,6 +150,7 @@ class Converter
         # Initialise empty lists for optional devices. These will be added
         # according to the user's selection
         @disks = []
+        @mounts = []
         @removables = []
         @nics = []
 
@@ -174,6 +185,20 @@ class Converter
         ], completion)
     end
 
+    def mount(dev, status, completion)
+        path  = "/dev/#{dev}"
+        status.call("Mounting #{dev}")
+        disk_type  = 'iscsi'
+        # TODO set uri
+        #      if nbd, use qemu-nbd to setup nbd access point to img
+        #      if iscsi, setup loop device infront of img, add new tgt
+        #            target to setup iscsi access point to loop device
+        uri = ""
+        iterate([
+            lambda { |cb| @connection.mount(uri, disk_type, &cb) }
+          ], completion)
+    end
+
     def iterate(stages, completion)
         i = 0
         cb = lambda { |result|
@@ -204,6 +229,13 @@ class Converter
                     'path'      => "/dev/#{device}"
                 }
             },
+            'mounts'    => @mounts.map { |dev|
+                {
+                    'device'     => device,
+                    'path'       => "/dev/#{device}",
+                    'uri'        => ""
+                }
+            },
             'removables' => @removables.map { |device|
                 removable = RemovableBlockDevice[device]
                 {
-- 
1.7.11.7

---
 p2v/server/virt-p2v-server.pl | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/p2v/server/virt-p2v-server.pl b/p2v/server/virt-p2v-server.pl
index c15efb2..f9dbc07 100755
--- a/p2v/server/virt-p2v-server.pl
+++ b/p2v/server/virt-p2v-server.pl
@@ -58,6 +58,7 @@ $SIG{'PIPE'} = 'IGNORE';
 use constant MSG_OPTIONS        => 'OPTIONS';
 use constant MSG_METADATA       => 'METADATA';
 use constant MSG_PATH           => 'PATH';
+use constant MSG_MOUNT          => 'MOUNT'
 use constant MSG_CONVERT        => 'CONVERT';
 use constant MSG_LIST_PROFILES  => 'LIST_PROFILES';
 use constant MSG_SET_PROFILE    => 'SET_PROFILE';
@@ -68,6 +69,7 @@ my @MSGS = (
     MSG_METADATA,
     MSG_OPTIONS,
     MSG_PATH,
+    MSG_MOUNT,
     MSG_CONVERT,
     MSG_LIST_PROFILES,
     MSG_SET_PROFILE,
@@ -181,6 +183,13 @@ eval {
             receive_path($path, $length);
         }
 
+        # MOUNT uri disk_type
+        elsif ($type eq MSG_MOUNT) {
+            my $uri = $msg->{args}[1];
+            my $disk_type = $msg->{args}[1];
+            mount_disk($uri, $disk_type);
+        }
+
         # CONVERT
         elsif ($type eq MSG_CONVERT) {
             convert();
@@ -358,6 +367,8 @@ sub convert
                             $_->{dst}->get_path(),
                             $_->{dst}->get_format() ] }
@{$meta->{disks}};
 
+        # TODO also add disks stored in $meta->{mounts}
+
         $g = new Sys::VirtConvert::GuestfsHandle(
             \@disks,
             $transferiso,
@@ -411,6 +422,16 @@ sub convert
     p2v_return_ok();
 }
 
+sub mount_disk
+{
+    my ($uri,$disk_type) = @_;
+
+    # TODO create local devices mapped to remote uri
+    #      if nbd use nbd-client, if iscsi use iscsiadm
+    #      store local device paths in $meta->{mounts}
+    p2v_return_ok();
+}
+
 sub unexpected_msg
 {
     my $msg = shift;
-- 
1.7.11.7