Libguestfs - Oct 2014 - [PATCH v5 REBASE 0/4] virt-resize: add support for resizing logical

Hi Rich,

This is rebase of v5 series. Meanwhile, I found a bug when shrinking
partitions, and the fix is incuded in this version (patch 2).

Regards,
Hu

changes to v4:
  1. add support to resize extended partition (--resize or --expand extended
partition)
  2. fix the problem of deficit of 512 bytes when expanding a logical partition
     (this problem can be reproduced in v4 by only expanding a logical
partition, without resizing any other partitions)
  3. update the test script to support logical partitions and extended partition

changes to v3:
  1. merge patch 1 and patch 3 in v3
  2. let mbr_part_type return 'primary' for GPT partitions
  3. add test for resizing logical partitions
  4. fix extending the extended partition (yet). see patch 7.

changes to v2:
  1. remove p_part_num
  2. remove filter_parts
  3. name the function calculate_target_partitions
  4. remove the code to restart guest introduced in v2

changes to v1:
  1. spit the patches so it's easier to review
  2. fix the parted error caused by unaligned logical partitions
  3. extend the content of logical partitions
  4. refactor to make logical partitions a seperate list



Hu Tao (4):
  resize: add partition type LogicalPartition
  resize: add support to resize logical partitions
  resize: support resize extended partition
  resize: test: add support for resizing extended and logical partitions

 resize/resize.ml           | 123 +++++++++++++++++++++++++++++++++++++++------
 resize/test-virt-resize.pl |  32 ++----------
 2 files changed, 112 insertions(+), 43 deletions(-)

-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index f54b11f..70f7e72 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -75,6 +75,7 @@ and partition_id  
 type partition_type    | PrimaryPartition
+  | LogicalPartition
 
 let rec debug_partition p    printf "%s:\n" p.p_name;
@@ -449,13 +450,15 @@ read the man page virt-resize(1).
   let find_partitions part_type      let parts = Array.to_list (g#part_list
"/dev/sda") in
 
-    (* Filter out logical partitions.  See note above. *)
     let parts        match part_type with
       (* for GPT, all partitions are regarded as Primary Partition,
        * e.g. there is no Extended Partition or Logical Partition. *)
       | PrimaryPartition ->
         List.filter (fun p -> parttype <> MBR || p.G.part_num <=
4_l)
+        parts
+      | LogicalPartition ->
+        List.filter (fun p -> parttype = MBR && p.G.part_num >=
5_l)
         parts in
 
     let partitions @@ -518,10 +521,12 @@ read the man page virt-resize(1).
     partitions in
 
   let partitions = find_partitions PrimaryPartition in
+  let logical_partitions = find_partitions LogicalPartition in
 
   if verbose then (
-    printf "%d partitions found\n" (List.length partitions);
-    List.iter debug_partition partitions
+    printf "%d partitions found\n" (List.length partitions +
List.length logical_partitions);
+    List.iter debug_partition partitions;
+    List.iter debug_partition logical_partitions
     );
 
   (* Build a data structure describing LVs on the source disk. *)
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 89 +++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 79 insertions(+), 10 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index 70f7e72..2655f87 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -596,6 +596,8 @@ read the man page virt-resize(1).
     let hash = Hashtbl.create 13 in
     List.iter (fun ({ p_name = name } as p) -> Hashtbl.add hash name p)
       partitions;
+    List.iter (fun ({ p_name = name } as p) -> Hashtbl.add hash name p)
+      logical_partitions;
     fun ~option name ->
       let name          if String.length name < 5 || String.sub name 0 5
<> "/dev/" then
@@ -719,8 +721,10 @@ read the man page virt-resize(1).
     (* We need some overhead for partitioning. *)
     let overhead        let maxl64 = List.fold_left max 0L in
+      let alignment = if alignment = 1L then 2L else alignment in
 
       let nr_partitions = List.length partitions in
+      let nr_partitions = nr_partitions + List.length logical_partitions in
 
       let gpt_start_sects = 64L in
       let gpt_end_sects = gpt_start_sects in
@@ -748,12 +752,25 @@ read the man page virt-resize(1).
     let required = List.fold_left (
       fun total p ->
         let newsize +          (* don't count extended partition but
logical partitions below,
+           * because we may extend and resize logical partitions at
+           * the same time. *)
+          if p.p_type = ContentExtendedPartition then 0L else
+            match p.p_operation with
+            | OpCopy | OpIgnore -> p.p_part.G.part_size
+            | OpDelete -> 0L
+            | OpResize newsize -> newsize in
+        total +^ newsize
+    ) 0L partitions in
+    let required = required +^ List.fold_left (
+      fun total p ->
+        let newsize            match p.p_operation with
           | OpCopy | OpIgnore -> p.p_part.G.part_size
           | OpDelete -> 0L
           | OpResize newsize -> newsize in
         total +^ newsize
-    ) 0L partitions in
+    ) 0L logical_partitions in
 
     let surplus = outsize -^ (required +^ overhead) in
 
@@ -799,6 +816,29 @@ read the man page virt-resize(1).
     )
   );
 
+  (* handle resizing of logical partitions *)
+  List.iter (
+    fun p ->
+      if p.p_type = ContentExtendedPartition then (
+        let alignment = if alignment = 1L then 2L else alignment in
+        let size = roundup64 p.p_part.G.part_size sectsize in
+        let logical_sizes = List.fold_left (
+          fun total p ->
+            match p.p_operation with
+              | OpDelete -> total +^ 0L
+              (* the start of logical partitions is aligned *)
+              | OpCopy | OpIgnore -> total +^ (roundup64
p.p_part.G.part_size (alignment *^ sectsize))
+              | OpResize newsize -> total +^ (roundup64 newsize (alignment
*^ sectsize))
+          ) 0L logical_partitions in
+        (* the first logical partition is aligned *)
+        let logical_sizes = logical_sizes +^ alignment *^ sectsize in
+        if logical_sizes > size then
+          p.p_operation <- OpResize logical_sizes
+        (* don't touch the extended partition if logical sizes less
+         * then the original size *)
+      )
+  ) partitions;
+
   (* Calculate the final surplus.
    * At this point, this number must be >= 0.
    *)
@@ -857,6 +897,7 @@ read the man page virt-resize(1).
       wrap (text ^ "\n\n") in
 
     List.iter print_summary partitions;
+    List.iter print_summary logical_partitions;
 
     List.iter (
       fun ({ lv_name = name } as lv) ->
@@ -1030,6 +1071,7 @@ read the man page virt-resize(1).
         calculate_target_partitions partnum start ~create_surplus ps (* skip p
*)
 
       | OpIgnore | OpCopy ->          (* same size *)
+          let start = roundup64 start 2L in
         (* Size in sectors. *)
         let size = div_roundup64 p.p_part.G.part_size sectsize in
         (* Start of next partition + alignment. *)
@@ -1045,17 +1087,25 @@ read the man page virt-resize(1).
           calculate_target_partitions (partnum+1) next ~create_surplus ps
 
       | OpResize newsize ->           (* resized partition *)
+          let start = roundup64 start 2L in
         (* New size in sectors. *)
         let size = div_roundup64 newsize sectsize in
         (* Start of next partition + alignment. *)
         let next = start +^ size in
         let next = roundup64 next alignment in
+        (* We will reserve a 1-sector gap between logical partitions
+         * by decreasing current partition by 1 sector, see below.
+         * But in case of shrink, we must ensure new partition size no
+         * less than newsize, thus make it one alignment-sectors larger. *)
+        let next = next +^ alignment in
 
         if verbose then
           printf "target partition %d: resize: newsize=%Ld start=%Ld
end=%Ld\n%!"
             partnum newsize start (next -^ 1L);
 
-        { p with p_target_start = start; p_target_end = next -^ 1L;
+          (* there must be a at least 1-sector gap between logical
+           * partitions otherwise parted refused to add logical partition *)
+          { p with p_target_start = start; p_target_end = next -^ 2L;
           p_target_partnum = partnum } ::
           calculate_target_partitions (partnum+1) next ~create_surplus ps
       )
@@ -1099,6 +1149,17 @@ read the man page virt-resize(1).
 
     calculate_target_partitions 1 start ~create_surplus:true partitions in
 
+  let logical_partitions +    let start = List.fold_left (
+      fun total p ->
+        match p.p_type with
+          | ContentExtendedPartition -> total +^ p.p_target_start
+          | _ -> total +^ 0L
+    ) 0L partitions in
+    (* align logical partitions, too *)
+    let start = roundup64 (start +^ 1L) alignment in
+    calculate_target_partitions 5 start ~create_surplus:false
logical_partitions in
+
   let mbr_part_type x      match parttype, x.p_part.G.part_num <= 4_l,
x.p_type with
     (* for GPT, all partitions are regarded as Primary Partition. *)
@@ -1114,6 +1175,11 @@ read the man page virt-resize(1).
       g#part_add "/dev/sdb" (mbr_part_type p) p.p_target_start
p.p_target_end
   ) partitions;
 
+  List.iter (
+    fun p ->
+      g#part_add "/dev/sdb" "logical" p.p_target_start
p.p_target_end
+  ) logical_partitions;
+
   (* Copy over the data. *)
   let copy_partition p        match p.p_operation with
@@ -1138,18 +1204,12 @@ read the man page virt-resize(1).
          | ContentUnknown | ContentPV _ | ContentFS _ ->
            g#copy_device_to_device ~size:copysize ~sparse source target
 
-         | ContentExtendedPartition ->
-           (* You can't just copy an extended partition by name, eg.
-            * source = "/dev/sda2", because the device name only
covers
-            * the first 1K of the partition.  Instead, copy the
-            * source bytes from the parent disk (/dev/sda).
-            *)
-           let srcoffset = p.p_part.G.part_start in
-           g#copy_device_to_device ~srcoffset ~size:copysize
"/dev/sda" target
+         | ContentExtendedPartition -> ()
         )
       | OpIgnore | OpDelete -> ()
   in
   List.iter copy_partition partitions;
+  List.iter copy_partition logical_partitions;
 
   (* Set bootable and MBR IDs.  Do this *after* copying over the data,
    * so that we can magically change the primary partition to an extended
@@ -1173,6 +1233,7 @@ read the man page virt-resize(1).
       | GPT, (No_ID|MBR_ID _) | MBR, (No_ID|GPT_Type _) -> ()
   in
   List.iter set_partition_bootable_and_id partitions;
+  List.iter set_partition_bootable_and_id logical_partitions;
 
   (* Fix the bootloader if we aligned the first partition. *)
   if align_first_partition_and_fix_bootloader then (
@@ -1224,6 +1285,13 @@ read the man page virt-resize(1).
         can_expand_content p.p_type
       | { p_operation = (OpCopy | OpIgnore | OpDelete) } -> false
     ) partitions
+    ||
+    List.exists (
+      function
+      | ({ p_operation = OpResize _ } as p) ->
+        can_expand_content p.p_type
+      | { p_operation = (OpCopy | OpIgnore | OpDelete) } -> false
+    ) logical_partitions
     || List.exists (
       function
       | ({ lv_operation = LVOpExpand } as lv) ->
@@ -1285,6 +1353,7 @@ read the man page virt-resize(1).
         -> ()
     in
     List.iter expand_partition_content partitions;
+    List.iter expand_partition_content logical_partitions;
 
     (* Expand logical volume content as required. *)
     List.iter (
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/resize.ml | 45 +++++++++++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index 2655f87..22fadc2 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -749,12 +749,33 @@ read the man page virt-resize(1).
         start_overhead_sects +^ alignment_sects +^ gpt_end_sects in
       sectsize *^ overhead_sects in
 
+    let required_logical = List.fold_left (
+      fun total p ->
+        let newsize +          match p.p_operation with
+          | OpCopy | OpIgnore -> roundup64 p.p_part.G.part_size (alignment
*^ sectsize)
+          | OpDelete -> 0L
+          | OpResize newsize -> roundup64 newsize (alignment *^ sectsize) in
+        total +^ newsize
+    ) 0L logical_partitions in
+    let required_logical = required_logical +^ alignment *^ sectsize in
+    let required_extended = List.fold_left (
+      fun total p ->
+        let newsize +        match p.p_type with
+        | ContentExtendedPartition ->
+          (* resizing extended partition by --resize or --expand is supported
*)
+          (match p.p_operation with
+           | OpResize newsize -> newsize
+           | _ -> p.p_part.G.part_size)
+        | _ -> 0L in
+        total +^ newsize
+    ) 0L partitions in
+
     let required = List.fold_left (
       fun total p ->
         let newsize -          (* don't count extended partition but
logical partitions below,
-           * because we may extend and resize logical partitions at
-           * the same time. *)
+          (* size of extended partition is calculated seperately *)
           if p.p_type = ContentExtendedPartition then 0L else
             match p.p_operation with
             | OpCopy | OpIgnore -> p.p_part.G.part_size
@@ -762,23 +783,17 @@ read the man page virt-resize(1).
             | OpResize newsize -> newsize in
         total +^ newsize
     ) 0L partitions in
-    let required = required +^ List.fold_left (
-      fun total p ->
-        let newsize -          match p.p_operation with
-          | OpCopy | OpIgnore -> p.p_part.G.part_size
-          | OpDelete -> 0L
-          | OpResize newsize -> newsize in
-        total +^ newsize
-    ) 0L logical_partitions in
-
+    let required = required +^ if required_extended > required_logical
+    then required_extended else required_logical in
     let surplus = outsize -^ (required +^ overhead) in
 
     if verbose then
       printf "calculate surplus: outsize=%Ld required=%Ld overhead=%Ld
surplus=%Ld\n%!"
         outsize required overhead surplus;
 
-    surplus
+    (* XXX: fix of expanding a logical partition. without the sectsize,
+     * we'll get a deficit of 512 bytes. *)
+    surplus +^ sectsize
   in
 
   (* Handle --expand and --shrink options. *)
@@ -822,6 +837,8 @@ read the man page virt-resize(1).
       if p.p_type = ContentExtendedPartition then (
         let alignment = if alignment = 1L then 2L else alignment in
         let size = roundup64 p.p_part.G.part_size sectsize in
+        (* resizing extended partition by --resize or --expand is supported *)
+        let size = match p.p_operation with OpResize s -> s | _ -> size
in
         let logical_sizes = List.fold_left (
           fun total p ->
             match p.p_operation with
-- 
1.9.3

Signed-off-by: Hu Tao <hutao@cn.fujitsu.com>
---
 resize/test-virt-resize.pl | 32 +++++---------------------------
 1 file changed, 5 insertions(+), 27 deletions(-)

diff --git a/resize/test-virt-resize.pl b/resize/test-virt-resize.pl
index a40f86c..544f274 100755
--- a/resize/test-virt-resize.pl
+++ b/resize/test-virt-resize.pl
@@ -75,13 +75,6 @@ if (rand () <= 0.5) {
 # an extended partition (#4) and zero or more logical partitions.
 my $nr_parts = 1 + int (rand (7));
 
-# XXX Temporarily restriction XXX
-# Currently virt-resize is broken when dealing with any extended
-# partition, so don't test this for the moment.
-if ($part_type eq "mbr" && $nr_parts >= 4) {
-    $nr_parts = 3;
-}
-
 # expand (1) or shrink (0)
 my $expand = 0;
 if (rand () >= 0.2) {
@@ -120,31 +113,16 @@ my $i;
 for ($i = 1; $i <= $nr_parts; ++$i) {
     $parts[$i] = { name => "sda".$i, resize => 0 };
 
-    if ($part_type eq "mbr") {
-        if ($i < 4) {
-            if (rand () < 0.5) {
-                $parts[$i]->{resize} = 1;
-            }
-        } elsif ($i == 4) {
+    if ($part_type eq "mbr" && $i == 4) {
             $parts[$i]->{content} = "extended";
-        }
-    } else {
-        if (rand () < 0.5) {
-            $parts[$i]->{resize} = 1;
-        }
+    }
+    if (rand () < 0.5) {
+        $parts[$i]->{resize} = 1;
     }
 }
 
 # Pick a partition at random to expand or shrink.
-if ($part_type eq "mbr") {
-    # virt-resize cannot shrink extended or logical partitions, so we
-    # set $max so that these cannot be chosen:
-    my $max = 3;
-    $max = $nr_parts if $max > $nr_parts;
-    $i = 1 + int (rand ($max));
-} else {
-    $i = 1 + int (rand ($nr_parts));
-}
+$i = 1 + int (rand ($nr_parts));
 $parts[$i]->{resize} = 0;
 $parts[$i]->{expand_shrink} = 1;
 
-- 
1.9.3

HI,

Any news?

Regards,
Hu

On Thu, Oct 30, 2014 at 10:46:52AM +0800, Hu Tao wrote:
> Hi Rich,
> 
> This is rebase of v5 series. Meanwhile, I found a bug when shrinking
> partitions, and the fix is incuded in this version (patch 2).
> 
> Regards,
> Hu
> 
> changes to v4:
>   1. add support to resize extended partition (--resize or --expand
extended partition)
>   2. fix the problem of deficit of 512 bytes when expanding a logical
partition
>      (this problem can be reproduced in v4 by only expanding a logical
partition, without resizing any other partitions)
>   3. update the test script to support logical partitions and extended
partition
> 
> changes to v3:
>   1. merge patch 1 and patch 3 in v3
>   2. let mbr_part_type return 'primary' for GPT partitions
>   3. add test for resizing logical partitions
>   4. fix extending the extended partition (yet). see patch 7.
> 
> changes to v2:
>   1. remove p_part_num
>   2. remove filter_parts
>   3. name the function calculate_target_partitions
>   4. remove the code to restart guest introduced in v2
> 
> changes to v1:
>   1. spit the patches so it's easier to review
>   2. fix the parted error caused by unaligned logical partitions
>   3. extend the content of logical partitions
>   4. refactor to make logical partitions a seperate list
> 
> 
> 
> Hu Tao (4):
>   resize: add partition type LogicalPartition
>   resize: add support to resize logical partitions
>   resize: support resize extended partition
>   resize: test: add support for resizing extended and logical partitions
> 
>  resize/resize.ml           | 123
+++++++++++++++++++++++++++++++++++++++------
>  resize/test-virt-resize.pl |  32 ++----------
>  2 files changed, 112 insertions(+), 43 deletions(-)
> 
> -- 
> 1.9.3
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs

ping again...

On Thu, Oct 30, 2014 at 10:46:52AM +0800, Hu Tao wrote:
> Hi Rich,
> 
> This is rebase of v5 series. Meanwhile, I found a bug when shrinking
> partitions, and the fix is incuded in this version (patch 2).
> 
> Regards,
> Hu
> 
> changes to v4:
>   1. add support to resize extended partition (--resize or --expand
extended partition)
>   2. fix the problem of deficit of 512 bytes when expanding a logical
partition
>      (this problem can be reproduced in v4 by only expanding a logical
partition, without resizing any other partitions)
>   3. update the test script to support logical partitions and extended
partition
> 
> changes to v3:
>   1. merge patch 1 and patch 3 in v3
>   2. let mbr_part_type return 'primary' for GPT partitions
>   3. add test for resizing logical partitions
>   4. fix extending the extended partition (yet). see patch 7.
> 
> changes to v2:
>   1. remove p_part_num
>   2. remove filter_parts
>   3. name the function calculate_target_partitions
>   4. remove the code to restart guest introduced in v2
> 
> changes to v1:
>   1. spit the patches so it's easier to review
>   2. fix the parted error caused by unaligned logical partitions
>   3. extend the content of logical partitions
>   4. refactor to make logical partitions a seperate list
> 
> 
> 
> Hu Tao (4):
>   resize: add partition type LogicalPartition
>   resize: add support to resize logical partitions
>   resize: support resize extended partition
>   resize: test: add support for resizing extended and logical partitions
> 
>  resize/resize.ml           | 123
+++++++++++++++++++++++++++++++++++++++------
>  resize/test-virt-resize.pl |  32 ++----------
>  2 files changed, 112 insertions(+), 43 deletions(-)
> 
> -- 
> 1.9.3
> 
> _______________________________________________
> Libguestfs mailing list
> Libguestfs@redhat.com
> https://www.redhat.com/mailman/listinfo/libguestfs

On Thu, Oct 30, 2014 at 10:46:52AM +0800, Hu Tao wrote:
> Hi Rich,
> 
> This is rebase of v5 series. Meanwhile, I found a bug when shrinking
> partitions, and the fix is incuded in this version (patch 2).
Hi,

When support to resizing logical partitions is enabled, there is a
problem about resizing the extended partition: whether the extended
partition is automatically resized (expanded or shrunk) by virt-resize
based on the resizing of logical partitions?

All the cases can be categorized as these two:

1. user doesn't explictly resize the extended partition. In this case it
   seems reasonable to automatically expand the extended partition as well.

2. user explictly resizes the extended partition. If the extended
   partition is resized as user requested, then we should deal with the
   conflicts about resizing extended partition and logical partitions,
   this is somewhat tricky in the current implementation; But if the
   extended partition is automatically resized, user will be confused as
   the resulting size may be what he/she expects.

Ideas?

Regards,
Hu

On Wed, Dec 17, 2014 at 03:07:11PM +0800, Hu Tao wrote:
> On Thu, Oct 30, 2014 at 10:46:52AM +0800, Hu Tao wrote:
> > Hi Rich,
> > 
> > This is rebase of v5 series. Meanwhile, I found a bug when shrinking
> > partitions, and the fix is incuded in this version (patch 2).
> 
> Hi,
> 
> When support to resizing logical partitions is enabled, there is a
> problem about resizing the extended partition: whether the extended
> partition is automatically resized (expanded or shrunk) by virt-resize
> based on the resizing of logical partitions?
> 
> All the cases can be categorized as these two:
> 
> 1. user doesn't explictly resize the extended partition. In this case
it
>    seems reasonable to automatically expand the extended partition as well.
> 
> 2. user explictly resizes the extended partition. If the extended
>    partition is resized as user requested, then we should deal with the
>    conflicts about resizing extended partition and logical partitions,
>    this is somewhat tricky in the current implementation; But if the
>    extended partition is automatically resized, user will be confused as
>    the resulting size may be what he/she expects.
s/may be/may not be/

I think the best choice is:

1. If user doesn't explictly resize the extended partition, then any
   resizing (expanding or shrinking) of logical partitions will cause
   the automatical resizing of the extended partition.

2. If user explictly resize the extended partition, then any confliction
   between resizing of the extended partition and logical partitions
   will raise an error.

To achieve this, the patchset needs to be refactored heavily. So I'd
like to hear suggestions before starting refactor.

Regards,
Hu