Libguestfs - Feb 2020 - [PATCH 0/1] virt-make-fs: add '--blocksize' option support

From: Nikolay Ivanets <stenavin@gmail.com>

This patch adds 4096 bytes sector size for output disk.

Side notes:

While working on this patch I reveal long standing issue: virt-make-fs can fail
if source directory/archive contains certain amount of really small or empty
files or wide tree of directories.  That is because of lack of available inodes
on a small file system to keep files structure.  In fact, the overall size of
the
source is not enought information to create minimal file system which can hold
all files and directories.  And in general it is hard-to-impossible to achive
this (while keeping file system as small as possible) for all possible file
systems without relying on internal knowlage about file system organization.
We do very basic adjustments in our estimations but that is not enough.

Nevertheless the above observations are not related to provided patch itself.

Nikolay Ivanets (1):
  virt-make-fs: add '--blocksize' option support

 make-fs/make-fs.c            | 28 ++++++++++++++++++++++++++--
 make-fs/test-virt-make-fs.sh |  5 ++++-
 make-fs/virt-make-fs.pod     | 10 ++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

-- 
2.17.2

From: Nikolay Ivanets <stenavin@gmail.com>

This patch adds '--blocksize' command line option for virt-make-fs
tool.  This option allows specifying disk sector size as described in
'guestfs_add_drive_opts' libguestfs API.
---
 make-fs/make-fs.c            | 28 ++++++++++++++++++++++++++--
 make-fs/test-virt-make-fs.sh |  5 ++++-
 make-fs/virt-make-fs.pod     | 10 ++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/make-fs/make-fs.c b/make-fs/make-fs.c
index 5d8c3a385..81d1cf90b 100644
--- a/make-fs/make-fs.c
+++ b/make-fs/make-fs.c
@@ -54,10 +54,12 @@ int in_virt_rescue = 0;
 
 static const char *format = "raw", *label = NULL,
   *partition = NULL, *size_str = NULL, *type = "ext2";
+int blocksize = 0;
 
 enum { HELP_OPTION = CHAR_MAX + 1 };
 static const char options[] = "F:s:t:Vvx";
 static const struct option long_options[] = {
+  { "blocksize", 1, 0, 0 },
   { "debug", 0, 0, 'v' }, /* for compat with Perl tool */
   { "floppy", 0, 0, 0 },
   { "format", 1, 0, 'F' },
@@ -87,6 +89,7 @@ usage (int status)
               "  %s [--options] input.tar.gz output.img\n"
               "  %s [--options] directory output.img\n"
               "Options:\n"
+              "  --blocksize=512|4096     Set sector size of the output
disk\n"
               "  --floppy                 Make a virtual floppy
disk\n"
               "  -F|--format=raw|qcow2|.. Set output format\n"
               "  --help                   Display brief help\n"
@@ -146,6 +149,9 @@ main (int argc, char *argv[])
           partition = "mbr";
         else
           partition = optarg;
+      } else if (STREQ (long_options[option_index].name,
"blocksize")) {
+        if (sscanf (optarg, "%d", &blocksize) != 1)
+          error (EXIT_FAILURE, 0, _("--blocksize option is not
numeric"));
       } else
         error (EXIT_FAILURE, 0,
                _("unknown long option: %s (%d)"),
@@ -678,11 +684,23 @@ do_make_fs (const char *input, const char *output_str)
              estimate, estimate / 1024, estimate / 4096);
   }
 
+  /* For partition alignment and extra space at the end of the disk:
+   * by default we create first partition at 128 sector boundary and leave the
+   * last 128 sectors at the end of the disk free.
+   */
+  if (partition)
+    estimate += 2 * 128 * blocksize;
+
   estimate += 256 * 1024;       /* For superblocks &c. */
 
   if (STRPREFIX (type, "ext") && type[3] >= '3') {
-    /* For ext3+, add some more for the journal. */
-    estimate += 1024 * 1024;
+    /* For ext3+, add some more for the journal.  Journal should be at least
+     * 1024 file system blocks.  By default file system block size is 1024
bytes
+     * if disk sector size is 512 bytes and 4096 bytes if disk sector size is
+     * 4096 bytes.  Note that default file system block size may be changed via
+     * /etc/mke2fs.conf but we cannot handle that at the moment.
+     */
+    estimate += 1024 * (blocksize == 512 ? 1024 : 4096);
   }
 
   else if (STREQ (type, "ntfs")) {
@@ -697,6 +715,11 @@ do_make_fs (const char *input, const char *output_str)
     estimate += 256 * 1024 * 1024;
   }
 
+  else if (STREQ (type, "xfs")) {
+    /* xfs requires at least 4096 blocks.  Default block size is 4096 bytes. */
+    estimate += 4096 * 4096;
+  }
+
   /* Add 10%, see above. */
   estimate *= 1.10;
 
@@ -718,6 +741,7 @@ do_make_fs (const char *input, const char *output_str)
 
   if (guestfs_add_drive_opts (g, output,
                               GUESTFS_ADD_DRIVE_OPTS_FORMAT, format,
+                              GUESTFS_ADD_DRIVE_OPTS_BLOCKSIZE, blocksize,
                               -1) == -1)
     return -1;
 
diff --git a/make-fs/test-virt-make-fs.sh b/make-fs/test-virt-make-fs.sh
index 79d058b6a..01c332b1d 100755
--- a/make-fs/test-virt-make-fs.sh
+++ b/make-fs/test-virt-make-fs.sh
@@ -83,9 +83,12 @@ size=`random_choice`
 choices=("" --label=FOO)
 label=`random_choice`
 
+choices=(--blocksize=512 --blocksize=4096)
+blocksize=`random_choice`
+
 if [ -n "$LIBGUESTFS_DEBUG" ]; then debug=--debug; fi
 
-params="$type $format $partition $size $label $debug"
+params="$type $format $partition $size $label $blocksize $debug"
 echo "test-virt-make-fs: parameters: $params"
 
 rm -f test.file test.tar output.img
diff --git a/make-fs/virt-make-fs.pod b/make-fs/virt-make-fs.pod
index 76ccbd120..bd4802c7c 100644
--- a/make-fs/virt-make-fs.pod
+++ b/make-fs/virt-make-fs.pod
@@ -122,6 +122,16 @@ or rerun virt-make-fs to build another image from scratch.
 
 Display brief help.
 
+=item B<--blocksize=512>
+
+=item B<--blocksize=4096>
+
+This parameter sets the sector size of the output disk image.
+
+The default is C<512> bytes.
+
+See also L<guestfs(3)/guestfs_add_drive_opts>.
+
 =item B<--floppy>
 
 Create a virtual floppy disk.
-- 
2.17.2

On Wed, Feb 19, 2020 at 02:09:29AM +0200, Mykola Ivanets
wrote:
> From: Nikolay Ivanets <stenavin@gmail.com>
> 
> This patch adds '--blocksize' command line option for virt-make-fs
> tool.  This option allows specifying disk sector size as described in
> 'guestfs_add_drive_opts' libguestfs API.
> ---
>  make-fs/make-fs.c            | 28 ++++++++++++++++++++++++++--
>  make-fs/test-virt-make-fs.sh |  5 ++++-
>  make-fs/virt-make-fs.pod     | 10 ++++++++++
>  3 files changed, 40 insertions(+), 3 deletions(-)
> 
> diff --git a/make-fs/make-fs.c b/make-fs/make-fs.c
> index 5d8c3a385..81d1cf90b 100644
> --- a/make-fs/make-fs.c
> +++ b/make-fs/make-fs.c
> @@ -54,10 +54,12 @@ int in_virt_rescue = 0;
>  
>  static const char *format = "raw", *label = NULL,
>    *partition = NULL, *size_str = NULL, *type = "ext2";
> +int blocksize = 0;
Needs to be a static int.
>  enum { HELP_OPTION = CHAR_MAX + 1 };
>  static const char options[] = "F:s:t:Vvx";
>  static const struct option long_options[] = {
> +  { "blocksize", 1, 0, 0 },
>    { "debug", 0, 0, 'v' }, /* for compat with Perl tool
*/
>    { "floppy", 0, 0, 0 },
>    { "format", 1, 0, 'F' },
> @@ -87,6 +89,7 @@ usage (int status)
>                "  %s [--options] input.tar.gz output.img\n"
>                "  %s [--options] directory output.img\n"
>                "Options:\n"
> +              "  --blocksize=512|4096     Set sector size of the
output disk\n"
>                "  --floppy                 Make a virtual floppy
disk\n"
>                "  -F|--format=raw|qcow2|.. Set output format\n"
>                "  --help                   Display brief help\n"
> @@ -146,6 +149,9 @@ main (int argc, char *argv[])
>            partition = "mbr";
>          else
>            partition = optarg;
> +      } else if (STREQ (long_options[option_index].name,
"blocksize")) {
> +        if (sscanf (optarg, "%d", &blocksize) != 1)
> +          error (EXIT_FAILURE, 0, _("--blocksize option is not
numeric"));
>        } else
>          error (EXIT_FAILURE, 0,
>                 _("unknown long option: %s (%d)"),
> @@ -678,11 +684,23 @@ do_make_fs (const char *input, const char
*output_str)
>               estimate, estimate / 1024, estimate / 4096);
>    }
>  
> +  /* For partition alignment and extra space at the end of the disk:
> +   * by default we create first partition at 128 sector boundary and leave
the
Best to keep lines under 78 or so characters long.  In emacs you can
use M-q (Esc q) to automatically format the comment.
> +   * last 128 sectors at the end of the disk free.
> +   */
> +  if (partition)
> +    estimate += 2 * 128 * blocksize;
> +
>    estimate += 256 * 1024;       /* For superblocks &c. */
>  
>    if (STRPREFIX (type, "ext") && type[3] >=
'3') {
> -    /* For ext3+, add some more for the journal. */
> -    estimate += 1024 * 1024;
> +    /* For ext3+, add some more for the journal.  Journal should be at
least
> +     * 1024 file system blocks.  By default file system block size is 1024
bytes
> +     * if disk sector size is 512 bytes and 4096 bytes if disk sector size
is
> +     * 4096 bytes.  Note that default file system block size may be
changed via
> +     * /etc/mke2fs.conf but we cannot handle that at the moment.
> +     */
Same here about the length of lines.
> +    estimate += 1024 * (blocksize == 512 ? 1024 : 4096);
>    }
>  
>    else if (STREQ (type, "ntfs")) {
> @@ -697,6 +715,11 @@ do_make_fs (const char *input, const char *output_str)
>      estimate += 256 * 1024 * 1024;
>    }
>  
> +  else if (STREQ (type, "xfs")) {
> +    /* xfs requires at least 4096 blocks.  Default block size is 4096
bytes. */
> +    estimate += 4096 * 4096;
> +  }
> +
>    /* Add 10%, see above. */
>    estimate *= 1.10;
>  
> @@ -718,6 +741,7 @@ do_make_fs (const char *input, const char *output_str)
>  
>    if (guestfs_add_drive_opts (g, output,
>                                GUESTFS_ADD_DRIVE_OPTS_FORMAT, format,
> +                              GUESTFS_ADD_DRIVE_OPTS_BLOCKSIZE, blocksize,
>                                -1) == -1)
>      return -1;
>  
> diff --git a/make-fs/test-virt-make-fs.sh b/make-fs/test-virt-make-fs.sh
> index 79d058b6a..01c332b1d 100755
> --- a/make-fs/test-virt-make-fs.sh
> +++ b/make-fs/test-virt-make-fs.sh
> @@ -83,9 +83,12 @@ size=`random_choice`
>  choices=("" --label=FOO)
>  label=`random_choice`
>  
> +choices=(--blocksize=512 --blocksize=4096)
> +blocksize=`random_choice`
> +
>  if [ -n "$LIBGUESTFS_DEBUG" ]; then debug=--debug; fi
>  
> -params="$type $format $partition $size $label $debug"
> +params="$type $format $partition $size $label $blocksize $debug"
>  echo "test-virt-make-fs: parameters: $params"
>  
>  rm -f test.file test.tar output.img
> diff --git a/make-fs/virt-make-fs.pod b/make-fs/virt-make-fs.pod
> index 76ccbd120..bd4802c7c 100644
> --- a/make-fs/virt-make-fs.pod
> +++ b/make-fs/virt-make-fs.pod
> @@ -122,6 +122,16 @@ or rerun virt-make-fs to build another image from
scratch.
>  
>  Display brief help.
>  
> +=item B<--blocksize=512>
> +
> +=item B<--blocksize=4096>
> +
> +This parameter sets the sector size of the output disk image.
> +
> +The default is C<512> bytes.
> +
> +See also L<guestfs(3)/guestfs_add_drive_opts>.
> +
>  =item B<--floppy>
>  
>  Create a virtual floppy disk.
Patch looks good apart from those few minor things.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
libguestfs lets you edit virtual machines.  Supports shell scripting,
bindings from many languages.  http://libguestfs.org