Libguestfs - Oct 2014 - [PATCH V5 1/4] resize: add partition type LogicalPartition

Hi Rich,

This is v5 series to add support for resizing MBR logical partitions. please
review. Thanks!

changes to v4:
  1. add support to resize extended partition (--resize or --expand extended
partition)
  2. fix the problem of deficit of 512 bytes when expanding a logical partition
     (this problem can be reproduced in v4 by only expanding a logical
partition, without resizing any other partitions)
  3. update the test script to support logical partitions and extended partition

changes to v3:
  1. merge patch 1 and patch 3 in v3
  2. let mbr_part_type return 'primary' for GPT partitions
  3. add test for resizing logical partitions
  4. fix extending the extended partition (yet). see patch 7.

changes to v2:
  1. remove p_part_num
  2. remove filter_parts
  3. name the function calculate_target_partitions
  4. remove the code to restart guest introduced in v2

changes to v1:
  1. spit the patches so it's easier to review
  2. fix the parted error caused by unaligned logical partitions
  3. extend the content of logical partitions
  4. refactor to make logical partitions a seperate list

Hu Tao (3):
  resize: add partition type LogicalPartition
  resize: add support to resize logical partitions
  resize: support resize extended partition

root (1):
  resize: test: add support for resizing extended and logical partitions

 resize/resize.ml           | 118 +++++++++++++++++++++++++++++++++++++++------
 resize/test-virt-resize.pl |  32 ++----------
 2 files changed, 107 insertions(+), 43 deletions(-)

-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index 2090675..cc76aa0 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -75,6 +75,7 @@ and partition_id  
 type partition_type    | PrimaryPartition
+  | LogicalPartition
 
 let rec debug_partition p    eprintf "%s:\n" p.p_name;
@@ -449,13 +450,15 @@ read the man page virt-resize(1).
   let find_partitions part_type      let parts = Array.to_list (g#part_list
"/dev/sda") in
 
-    (* Filter out logical partitions.  See note above. *)
     let parts        match part_type with
       (* for GPT, all partitions are regarded as Primary Partition,
        * e.g. there is no Extended Partition or Logical Partition. *)
       | PrimaryPartition ->
         List.filter (fun p -> parttype <> MBR || p.G.part_num <=
4_l)
+        parts
+      | LogicalPartition ->
+        List.filter (fun p -> parttype = MBR && p.G.part_num >=
5_l)
         parts in
 
     let partitions @@ -518,10 +521,12 @@ read the man page virt-resize(1).
     partitions in
 
   let partitions = find_partitions PrimaryPartition in
+  let logical_partitions = find_partitions LogicalPartition in
 
   if verbose then (
-    eprintf "%d partitions found\n" (List.length partitions);
-    List.iter debug_partition partitions
+    eprintf "%d partitions found\n" (List.length partitions +
List.length logical_partitions);
+    List.iter debug_partition partitions;
+    List.iter debug_partition logical_partitions
     );
 
   (* Build a data structure describing LVs on the source disk. *)
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 84 +++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 74 insertions(+), 10 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index cc76aa0..fc622ba 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -596,6 +596,8 @@ read the man page virt-resize(1).
     let hash = Hashtbl.create 13 in
     List.iter (fun ({ p_name = name } as p) -> Hashtbl.add hash name p)
       partitions;
+    List.iter (fun ({ p_name = name } as p) -> Hashtbl.add hash name p)
+      logical_partitions;
     fun ~option name ->
       let name          if String.length name < 5 || String.sub name 0 5
<> "/dev/" then
@@ -719,8 +721,10 @@ read the man page virt-resize(1).
     (* We need some overhead for partitioning. *)
     let overhead        let maxl64 = List.fold_left max 0L in
+      let alignment = if alignment = 1L then 2L else alignment in
 
       let nr_partitions = List.length partitions in
+      let nr_partitions = nr_partitions + List.length logical_partitions in
 
       let gpt_start_sects = 64L in
       let gpt_end_sects = gpt_start_sects in
@@ -748,12 +752,25 @@ read the man page virt-resize(1).
     let required = List.fold_left (
       fun total p ->
         let newsize +          (* don't count extended partition but
logical partitions below,
+           * because we may extend and resize logical partitions at
+           * the same time. *)
+          if p.p_type = ContentExtendedPartition then 0L else
+            match p.p_operation with
+            | OpCopy | OpIgnore -> p.p_part.G.part_size
+            | OpDelete -> 0L
+            | OpResize newsize -> newsize in
+        total +^ newsize
+    ) 0L partitions in
+    let required = required +^ List.fold_left (
+      fun total p ->
+        let newsize            match p.p_operation with
           | OpCopy | OpIgnore -> p.p_part.G.part_size
           | OpDelete -> 0L
           | OpResize newsize -> newsize in
         total +^ newsize
-    ) 0L partitions in
+    ) 0L logical_partitions in
 
     let surplus = outsize -^ (required +^ overhead) in
 
@@ -799,6 +816,29 @@ read the man page virt-resize(1).
     )
   );
 
+  (* handle resizing of logical partitions *)
+  List.iter (
+    fun p ->
+      if p.p_type = ContentExtendedPartition then (
+        let alignment = if alignment = 1L then 2L else alignment in
+        let size = roundup64 p.p_part.G.part_size sectsize in
+        let logical_sizes = List.fold_left (
+          fun total p ->
+            match p.p_operation with
+              | OpDelete -> total +^ 0L
+              (* the start of logical partitions is aligned *)
+              | OpCopy | OpIgnore -> total +^ (roundup64
p.p_part.G.part_size (alignment *^ sectsize))
+              | OpResize newsize -> total +^ (roundup64 newsize (alignment
*^ sectsize))
+          ) 0L logical_partitions in
+        (* the first logical partition is aligned *)
+        let logical_sizes = logical_sizes +^ alignment *^ sectsize in
+        if logical_sizes > size then
+          p.p_operation <- OpResize logical_sizes
+        (* don't touch the extended partition if logical sizes less
+         * then the original size *)
+      )
+  ) partitions;
+
   (* Calculate the final surplus.
    * At this point, this number must be >= 0.
    *)
@@ -857,6 +897,7 @@ read the man page virt-resize(1).
         wrap ~indent:4 (text ^ "\n\n") in
 
     List.iter print_summary partitions;
+    List.iter print_summary logical_partitions;
 
     List.iter (
       fun ({ lv_name = name } as lv) ->
@@ -1031,6 +1072,7 @@ read the man page virt-resize(1).
         calculate_target_partitions partnum start ~create_surplus ps (* skip p
*)
 
       | OpIgnore | OpCopy ->          (* same size *)
+          let start = roundup64 start 2L in
         (* Size in sectors. *)
         let size = div_roundup64 p.p_part.G.part_size sectsize in
         (* Start of next partition + alignment. *)
@@ -1046,6 +1088,7 @@ read the man page virt-resize(1).
           calculate_target_partitions (partnum+1) next ~create_surplus ps
 
       | OpResize newsize ->           (* resized partition *)
+          let start = roundup64 start 2L in
         (* New size in sectors. *)
         let size = div_roundup64 newsize sectsize in
         (* Start of next partition + alignment. *)
@@ -1056,7 +1099,9 @@ read the man page virt-resize(1).
           eprintf "target partition %d: resize: newsize=%Ld start=%Ld
end=%Ld\n%!"
             partnum newsize start (next -^ 1L);
 
-        { p with p_target_start = start; p_target_end = next -^ 1L;
+          (* there must be a at least 1-sector gap between logical
+           * partitions otherwise parted refused to add logical partition *)
+          { p with p_target_start = start; p_target_end = next -^ 2L;
           p_target_partnum = partnum } ::
           calculate_target_partitions (partnum+1) next ~create_surplus ps
       )
@@ -1100,6 +1145,17 @@ read the man page virt-resize(1).
 
     calculate_target_partitions 1 start ~create_surplus:true partitions in
 
+  let logical_partitions +    let start = List.fold_left (
+      fun total p ->
+        match p.p_type with
+          | ContentExtendedPartition -> total +^ p.p_target_start
+          | _ -> total +^ 0L
+    ) 0L partitions in
+    (* align logical partitions, too *)
+    let start = roundup64 (start +^ 1L) alignment in
+    calculate_target_partitions 5 start ~create_surplus:false
logical_partitions in
+
   let mbr_part_type x      match parttype, x.p_part.G.part_num <= 4_l,
x.p_type with
     (* for GPT, all partitions are regarded as Primary Partition. *)
@@ -1115,6 +1171,11 @@ read the man page virt-resize(1).
       g#part_add "/dev/sdb" (mbr_part_type p) p.p_target_start
p.p_target_end
   ) partitions;
 
+  List.iter (
+    fun p ->
+      g#part_add "/dev/sdb" "logical" p.p_target_start
p.p_target_end
+  ) logical_partitions;
+
   (* Copy over the data. *)
   let copy_partition p        match p.p_operation with
@@ -1140,18 +1201,12 @@ read the man page virt-resize(1).
          | ContentUnknown | ContentPV _ | ContentFS _ ->
            g#copy_device_to_device ~size:copysize ~sparse source target
 
-         | ContentExtendedPartition ->
-           (* You can't just copy an extended partition by name, eg.
-            * source = "/dev/sda2", because the device name only
covers
-            * the first 1K of the partition.  Instead, copy the
-            * source bytes from the parent disk (/dev/sda).
-            *)
-           let srcoffset = p.p_part.G.part_start in
-           g#copy_device_to_device ~srcoffset ~size:copysize
"/dev/sda" target
+         | ContentExtendedPartition -> ()
         )
       | OpIgnore | OpDelete -> ()
   in
   List.iter copy_partition partitions;
+  List.iter copy_partition logical_partitions;
 
   (* Set bootable and MBR IDs.  Do this *after* copying over the data,
    * so that we can magically change the primary partition to an extended
@@ -1175,6 +1230,7 @@ read the man page virt-resize(1).
       | GPT, (No_ID|MBR_ID _) | MBR, (No_ID|GPT_Type _) -> ()
   in
   List.iter set_partition_bootable_and_id partitions;
+  List.iter set_partition_bootable_and_id logical_partitions;
 
   (* Fix the bootloader if we aligned the first partition. *)
   if align_first_partition_and_fix_bootloader then (
@@ -1227,6 +1283,13 @@ read the man page virt-resize(1).
         can_expand_content p.p_type
       | { p_operation = (OpCopy | OpIgnore | OpDelete) } -> false
     ) partitions
+    ||
+    List.exists (
+      function
+      | ({ p_operation = OpResize _ } as p) ->
+        can_expand_content p.p_type
+      | { p_operation = (OpCopy | OpIgnore | OpDelete) } -> false
+    ) logical_partitions
     || List.exists (
       function
       | ({ lv_operation = LVOpExpand } as lv) ->
@@ -1289,6 +1352,7 @@ read the man page virt-resize(1).
         -> ()
     in
     List.iter expand_partition_content partitions;
+    List.iter expand_partition_content logical_partitions;
 
     (* Expand logical volume content as required. *)
     List.iter (
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 45 +++++++++++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index fc622ba..80a37e2 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -749,12 +749,33 @@ read the man page virt-resize(1).
         start_overhead_sects +^ alignment_sects +^ gpt_end_sects in
       sectsize *^ overhead_sects in
 
+    let required_logical = List.fold_left (
+      fun total p ->
+        let newsize +          match p.p_operation with
+          | OpCopy | OpIgnore -> roundup64 p.p_part.G.part_size (alignment
*^ sectsize)
+          | OpDelete -> 0L
+          | OpResize newsize -> roundup64 newsize (alignment *^ sectsize) in
+        total +^ newsize
+    ) 0L logical_partitions in
+    let required_logical = required_logical +^ alignment *^ sectsize in
+    let required_extended = List.fold_left (
+      fun total p ->
+        let newsize +        match p.p_type with
+        | ContentExtendedPartition ->
+          (* resizing extended partition by --resize or --expand is supported
*)
+          (match p.p_operation with
+           | OpResize newsize -> newsize
+           | _ -> p.p_part.G.part_size)
+        | _ -> 0L in
+        total +^ newsize
+    ) 0L partitions in
+
     let required = List.fold_left (
       fun total p ->
         let newsize -          (* don't count extended partition but
logical partitions below,
-           * because we may extend and resize logical partitions at
-           * the same time. *)
+          (* size of extended partition is calculated seperately *)
           if p.p_type = ContentExtendedPartition then 0L else
             match p.p_operation with
             | OpCopy | OpIgnore -> p.p_part.G.part_size
@@ -762,23 +783,17 @@ read the man page virt-resize(1).
             | OpResize newsize -> newsize in
         total +^ newsize
     ) 0L partitions in
-    let required = required +^ List.fold_left (
-      fun total p ->
-        let newsize -          match p.p_operation with
-          | OpCopy | OpIgnore -> p.p_part.G.part_size
-          | OpDelete -> 0L
-          | OpResize newsize -> newsize in
-        total +^ newsize
-    ) 0L logical_partitions in
-
+    let required = required +^ if required_extended > required_logical
+    then required_extended else required_logical in
     let surplus = outsize -^ (required +^ overhead) in
 
     if verbose then
       eprintf "calculate surplus: outsize=%Ld required=%Ld overhead=%Ld
surplus=%Ld\n%!"
         outsize required overhead surplus;
 
-    surplus
+    (* XXX: fix of expanding a logical partition. without the sectsize,
+     * we'll get a deficit of 512 bytes. *)
+    surplus +^ sectsize
   in
 
   (* Handle --expand and --shrink options. *)
@@ -822,6 +837,8 @@ read the man page virt-resize(1).
       if p.p_type = ContentExtendedPartition then (
         let alignment = if alignment = 1L then 2L else alignment in
         let size = roundup64 p.p_part.G.part_size sectsize in
+        (* resizing extended partition by --resize or --expand is supported *)
+        let size = match p.p_operation with OpResize s -> s | _ -> size
in
         let logical_sizes = List.fold_left (
           fun total p ->
             match p.p_operation with
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/test-virt-resize.pl | 32 +++++---------------------------
 1 file changed, 5 insertions(+), 27 deletions(-)

diff --git a/resize/test-virt-resize.pl b/resize/test-virt-resize.pl
index a40f86c..544f274 100755
--- a/resize/test-virt-resize.pl
+++ b/resize/test-virt-resize.pl
@@ -75,13 +75,6 @@ if (rand () <= 0.5) {
 # an extended partition (#4) and zero or more logical partitions.
 my $nr_parts = 1 + int (rand (7));
 
-# XXX Temporarily restriction XXX
-# Currently virt-resize is broken when dealing with any extended
-# partition, so don't test this for the moment.
-if ($part_type eq "mbr" && $nr_parts >= 4) {
-    $nr_parts = 3;
-}
-
 # expand (1) or shrink (0)
 my $expand = 0;
 if (rand () >= 0.2) {
@@ -120,31 +113,16 @@ my $i;
 for ($i = 1; $i <= $nr_parts; ++$i) {
     $parts[$i] = { name => "sda".$i, resize => 0 };
 
-    if ($part_type eq "mbr") {
-        if ($i < 4) {
-            if (rand () < 0.5) {
-                $parts[$i]->{resize} = 1;
-            }
-        } elsif ($i == 4) {
+    if ($part_type eq "mbr" && $i == 4) {
             $parts[$i]->{content} = "extended";
-        }
-    } else {
-        if (rand () < 0.5) {
-            $parts[$i]->{resize} = 1;
-        }
+    }
+    if (rand () < 0.5) {
+        $parts[$i]->{resize} = 1;
     }
 }
 
 # Pick a partition at random to expand or shrink.
-if ($part_type eq "mbr") {
-    # virt-resize cannot shrink extended or logical partitions, so we
-    # set $max so that these cannot be chosen:
-    my $max = 3;
-    $max = $nr_parts if $max > $nr_parts;
-    $i = 1 + int (rand ($max));
-} else {
-    $i = 1 + int (rand ($nr_parts));
-}
+$i = 1 + int (rand ($nr_parts));
 $parts[$i]->{resize} = 0;
 $parts[$i]->{expand_shrink} = 1;
 
-- 
1.9.3

ping...

On Wed, Oct 08, 2014 at 05:11:13PM +0800, Hu Tao wrote:
> Hi Rich,
> 
> This is v5 series to add support for resizing MBR logical partitions.
please review. Thanks!
> 
> changes to v4:
>   1. add support to resize extended partition (--resize or --expand
extended partition)
>   2. fix the problem of deficit of 512 bytes when expanding a logical
partition
>      (this problem can be reproduced in v4 by only expanding a logical
partition, without resizing any other partitions)
>   3. update the test script to support logical partitions and extended
partition
> 
> changes to v3:
>   1. merge patch 1 and patch 3 in v3
>   2. let mbr_part_type return 'primary' for GPT partitions
>   3. add test for resizing logical partitions
>   4. fix extending the extended partition (yet). see patch 7.
> 
> changes to v2:
>   1. remove p_part_num
>   2. remove filter_parts
>   3. name the function calculate_target_partitions
>   4. remove the code to restart guest introduced in v2
> 
> changes to v1:
>   1. spit the patches so it's easier to review
>   2. fix the parted error caused by unaligned logical partitions
>   3. extend the content of logical partitions
>   4. refactor to make logical partitions a seperate list
> 
> Hu Tao (3):
>   resize: add partition type LogicalPartition
>   resize: add support to resize logical partitions
>   resize: support resize extended partition
> 
> root (1):
>   resize: test: add support for resizing extended and logical partitions
> 
>  resize/resize.ml           | 118
+++++++++++++++++++++++++++++++++++++++------
>  resize/test-virt-resize.pl |  32 ++----------
>  2 files changed, 107 insertions(+), 43 deletions(-)
> 
> -- 
> 1.9.3
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs

On Tue, Oct 21, 2014 at 10:00:10AM +0800, Hu Tao wrote:
> ping...
I tried to look at this yesterday.  It's very complex and I want to
give it proper consideration when I'm not so very busy on virt-v2v.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v