Libguestfs - Oct 2020 - Re: [PATCH nbdinfo v2] info: Add a --map option for displaying allocation metadata.

Fixes bugs with JSON output in the previous patch, and adds some tests.

Rich.

---
 info/Makefile.am                      |   4 +
 info/info-map-base-allocation-json.sh |  52 ++++++++++
 info/info-map-base-allocation.sh      |  49 ++++++++++
 info/nbdinfo.c                        | 134 ++++++++++++++++++++++++--
 info/nbdinfo.pod                      |  39 +++++++-
 5 files changed, 271 insertions(+), 7 deletions(-)

diff --git a/info/Makefile.am b/info/Makefile.am
index 5d0d288..496692a 100644
--- a/info/Makefile.am
+++ b/info/Makefile.am
@@ -25,6 +25,8 @@ EXTRA_DIST = \
 	info-size.sh \
 	info-text.sh \
 	info-description.sh \
+	info-map-base-allocation.sh \
+	info-map-base-allocation-json.sh \
 	nbdinfo.pod \
 	$(NULL)
 
@@ -67,6 +69,8 @@ TESTS += \
 	info-size.sh \
 	info-text.sh \
 	info-description.sh \
+	info-map-base-allocation.sh \
+	info-map-base-allocation-json.sh \
 	$(NULL)
 
 check-valgrind:
diff --git a/info/info-map-base-allocation-json.sh
b/info/info-map-base-allocation-json.sh
new file mode 100755
index 0000000..7927178
--- /dev/null
+++ b/info/info-map-base-allocation-json.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# nbd client library in userspace
+# Copyright (C) 2020 Red Hat Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+. ../tests/functions.sh
+
+set -e
+set -x
+
+requires nbdkit --version
+requires nbdsh --version
+requires jq --version
+
+out=info-base-allocation-json.out
+cleanup_fn rm -f $out
+rm -f $out
+
+# Note the memory plugin uses a 32K page size, and extents
+# are always aligned with this.
+nbdkit -U - memory 1M --run '
+    nbdsh -u "$uri" \
+          -c "h.pwrite(b\"\\x01\"*131072, 0)" \
+          -c "h.pwrite(b\"\\x02\"*131072, 320*1024)"
&&
+    nbdinfo --map --json "$uri"
+' > $out
+
+cat $out
+jq < $out
+
+test $( jq -r '.[0].offset' < $out ) -eq 0
+test $( jq -r '.[0].length' < $out ) -eq 131072
+test $( jq -r '.[0].type' < $out ) -eq 0
+test $( jq -r '.[0].description' < $out ) = "allocated"
+
+test $( jq -r '.[3].offset' < $out ) -eq 458752
+test $( jq -r '.[3].length' < $out ) -eq 589824
+test $( jq -r '.[3].type' < $out ) -eq 3
+test $( jq -r '.[3].description' < $out ) = "hole,zero"
diff --git a/info/info-map-base-allocation.sh b/info/info-map-base-allocation.sh
new file mode 100755
index 0000000..104ed7c
--- /dev/null
+++ b/info/info-map-base-allocation.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# nbd client library in userspace
+# Copyright (C) 2020 Red Hat Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+. ../tests/functions.sh
+
+set -e
+set -x
+
+requires nbdkit --version
+requires nbdsh --version
+requires tr --version
+
+out=info-base-allocation.out
+cleanup_fn rm -f $out
+rm -f $out
+
+# Note the memory plugin uses a 32K page size, and extents
+# are always aligned with this.
+nbdkit -U - memory 1M --run '
+    nbdsh -u "$uri" \
+          -c "h.pwrite(b\"\\x01\"*131072, 0)" \
+          -c "h.pwrite(b\"\\x02\"*131072, 320*1024)"
&&
+    nbdinfo --map "$uri"
+' > $out
+
+cat $out
+
+if [ "$(tr -s ' ' < $out)" != " 0 131072 0 allocated
+ 131072 196608 3 hole,zero
+ 327680 131072 0 allocated
+ 458752 589824 3 hole,zero" ]; then
+    echo "$0: unexpected output from nbdinfo --map"
+    exit 1
+fi
diff --git a/info/nbdinfo.c b/info/nbdinfo.c
index 647a24c..ff060aa 100644
--- a/info/nbdinfo.c
+++ b/info/nbdinfo.c
@@ -34,6 +34,7 @@
 static bool list_all = false;
 static bool probe_content, content_flag, no_content_flag;
 static bool json_output = false;
+static const char *map = NULL;
 static bool size_only = false;
 
 static struct export_list {
@@ -49,6 +50,10 @@ static void list_one_export (struct nbd_handle *nbd, const
char *desc,
 static void list_all_exports (struct nbd_handle *nbd1, const char *uri);
 static void print_json_string (const char *);
 static char *get_content (struct nbd_handle *, int64_t size);
+static int extent_callback (void *user_data, const char *metacontext,
+                            uint64_t offset,
+                            uint32_t *entries, size_t nr_entries,
+                            int *error);
 
 static void __attribute__((noreturn))
 usage (FILE *fp, int exitcode)
@@ -60,6 +65,7 @@ usage (FILE *fp, int exitcode)
 "    nbdinfo nbd://localhost\n"
 "    nbdinfo \"nbd+unix:///?socket=/tmp/unixsock\"\n"
 "    nbdinfo --size nbd://example.com\n"
+"    nbdinfo --map nbd://example.com\n"
 "    nbdinfo --json nbd://example.com\n"
 "    nbdinfo --list nbd://example.com\n"
 "\n"
@@ -85,6 +91,7 @@ main (int argc, char *argv[])
     CONTENT_OPTION,
     NO_CONTENT_OPTION,
     JSON_OPTION,
+    MAP_OPTION,
     SIZE_OPTION,
   };
   const char *short_options = "LV";
@@ -95,6 +102,7 @@ main (int argc, char *argv[])
     { "json",               no_argument,       NULL, JSON_OPTION },
     { "list",               no_argument,       NULL, 'L' },
     { "long-options",       no_argument,       NULL, LONG_OPTIONS },
+    { "map",                optional_argument, NULL, MAP_OPTION },
     { "short-options",      no_argument,       NULL, SHORT_OPTIONS },
     { "size",               no_argument,       NULL, SIZE_OPTION },
     { "version",            no_argument,       NULL, 'V' },
@@ -143,6 +151,10 @@ main (int argc, char *argv[])
       no_content_flag = true;
       break;
 
+    case MAP_OPTION:
+      map = optarg ? optarg : "base:allocation";
+      break;
+
     case SIZE_OPTION:
       size_only = true;
       break;
@@ -164,10 +176,11 @@ main (int argc, char *argv[])
   if (argc - optind != 1)
     usage (stderr, EXIT_FAILURE);
 
-  /* You can combine certain options. */
-  if (list_all && size_only) {
-    fprintf (stderr, "%s: you cannot use %s and %s together.\n",
-             argv[0], "--list", "--size");
+  /* You cannot combine certain options. */
+  if (!!list_all + !!map + !!size_only > 1) {
+    fprintf (stderr,
+             "%s: you cannot use --list, --map and --size
together.\n",
+             argv[0]);
     exit (EXIT_FAILURE);
   }
   if (content_flag && no_content_flag) {
@@ -182,6 +195,8 @@ main (int argc, char *argv[])
     probe_content = true;
   if (no_content_flag)
     probe_content = false;
+  if (map)
+    probe_content = false;
 
   /* Open the NBD side. */
   nbd = nbd_create ();
@@ -191,11 +206,13 @@ main (int argc, char *argv[])
   }
   nbd_set_uri_allow_local_file (nbd, true); /* Allow ?tls-psk-file. */
 
-  /* If using --list then we need opt mode in the handle. */
+  /* Set optional modes in the handle. */
   if (list_all)
     nbd_set_opt_mode (nbd, true);
-  if (!size_only)
+  if (!map && !size_only)
     nbd_set_full_info (nbd, true);
+  if (map)
+    nbd_add_meta_context (nbd, map);
 
   if (nbd_connect_uri (nbd, argv[optind]) == -1) {
     fprintf (stderr, "%s\n", nbd_get_error ());
@@ -225,6 +242,44 @@ main (int argc, char *argv[])
 
     printf ("%" PRIi64 "\n", size);
   }
+  else if (map) {
+    uint64_t offset, prev_offset;
+
+    /* Did we get the requested map? */
+    if (!nbd_can_meta_context (nbd, map)) {
+      fprintf (stderr,
+               "%s: --map: server does not support metadata context
\"%s\"\n",
+               argv[0], map);
+      exit (EXIT_FAILURE);
+    }
+
+    size = nbd_get_size (nbd);
+    if (size == -1) {
+      fprintf (stderr, "%s\n", nbd_get_error ());
+      exit (EXIT_FAILURE);
+    }
+
+    if (json_output) printf ("[\n");
+    for (offset = 0; offset < size;) {
+      prev_offset = offset;
+      if (nbd_block_status (nbd, size - offset, offset,
+                            (nbd_extent_callback) { .callback =
extent_callback,
+                                                    .user_data = &offset },
+                            0) == -1) {
+        fprintf (stderr, "%s\n", nbd_get_error ());
+        exit (EXIT_FAILURE);
+      }
+      /* We expect extent_callback to increment the offset.  If it did
+       * not then probably the server is not returning any extents.
+       */
+      if (offset <= prev_offset) {
+        fprintf (stderr, "%s: --map: server did not return any
extents\n",
+                 argv[0]);
+        exit (EXIT_FAILURE);
+      }
+    }
+    if (json_output) printf ("\n]\n");
+  }
   else {
     /* Print per-connection fields. */
     protocol = nbd_get_protocol (nbd);
@@ -591,3 +646,70 @@ get_content (struct nbd_handle *nbd, int64_t size)
   free (cmd);
   return ret;                   /* caller frees */
 }
+
+/* Callback handling --map. */
+static const char *
+extent_description (const char *metacontext, uint32_t type)
+{
+  if (strcmp (metacontext, "base:allocation") == 0) {
+    switch (type) {
+    case 0: return "allocated";
+    case 1: return "zero";
+    case 2: return "hole";
+    case 3: return "hole,zero";
+    }
+  }
+  else if (strcmp (metacontext, "qemu:dirty-bitmap") == 0) {
+    switch (type) {
+    case 0: return "clean";
+    case 1: return "dirty";
+    }
+  }
+
+  return "unknown";
+}
+
+static int
+extent_callback (void *user_data, const char *metacontext,
+                 uint64_t offset,
+                 uint32_t *entries, size_t nr_entries,
+                 int *error)
+{
+  size_t i;
+  uint64_t *ret_offset = user_data;
+  static bool comma = false;
+
+  if (strcmp (metacontext, map) != 0)
+    return 0;
+
+  /* Print the entries received. */
+  for (i = 0; i < nr_entries; i += 2) {
+    const char *descr = extent_description (map, entries[i+1]);
+
+    if (!json_output) {
+      printf ("%10" PRIu64 "  "
+              "%10" PRIu32 "  "
+              "%3" PRIu32 "  "
+              "%s\n",
+              offset, entries[i], entries[i+1], descr);
+    }
+    else {
+      if (comma)
+        printf (",\n");
+
+      printf ("{ \"offset\": %" PRIu64 ", "
+              "\"length\": %" PRIu32 ", "
+              "\"type\": %" PRIu32 ", ",
+              offset, entries[i], entries[i+1]);
+      printf ("\"description\": ");
+      print_json_string (descr);
+      printf ("}");
+      comma = true;
+    }
+
+    offset += entries[i];
+  }
+
+  *ret_offset = offset;
+  return 0;
+}
diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod
index 19305bf..0e65f1f 100644
--- a/info/nbdinfo.pod
+++ b/info/nbdinfo.pod
@@ -4,7 +4,7 @@ nbdinfo - display information and metadata about NBD servers and
exports
 
 =head1 SYNOPSIS
 
- nbdinfo [--json] [--size] NBD-URI
+ nbdinfo [--json] [--map] [--size] NBD-URI
 
  nbdinfo -L|--list NBD-URI
 
@@ -20,6 +20,8 @@ nbdinfo - display information and metadata about NBD servers
and exports
 
  nbdinfo --size nbd://example.com
 
+ nbdinfo --map nbd://example.com
+
  nbdinfo --json nbd://example.com
 
  nbdinfo --list nbd://example.com
@@ -84,6 +86,32 @@ the I<--json> parameter:
    ]
  }
 
+=head3 Map
+
+To show a map which areas of the disk are allocated and sparse, use
+the I<--map> option:
+
+ $ nbdinfo --map nbd://localhost/
+ 0        1048576  0  allocated
+ 1048576  1048576  3  hole,zero
+
+The fields are: start size type description.
+
+The type field is an integer showing the raw value from the NBD
+protocol.  For some maps nbdinfo knows how to translate the type into
+a printable description.
+
+By default this shows the C<"base:allocation"> map, but you can
show
+other maps too:
+
+ $ nbdinfo --map=qemu:dirty-bitmap nbd://localhost/
+ 0  1048576  1  dirty
+
+For more information on NBD maps, see I<Metadata querying> in the NBD
+protocol.  I<--json> can also be used here for parsable JSON output.
+
+=head3 List of exports
+
 To list all the exports available on an NBD server use the I<--list>
 (I<-L>) option.
 
@@ -122,6 +150,15 @@ use I<--list --content>.
 
 The output is displayed in JSON format.
 
+=item B<--map>
+
+=item B<--map=>MAP
+
+Display the map (usually whether parts of the disk are allocated or
+sparse) of the given export.  This displays the
C<"base:allocation">
+map by default, you can choose a different map with the optional
+parameter.
+
 =item B<-L>
 
 =item B<--list>
-- 
2.27.0

On 9/26/20 11:36 AM, Richard W.M. Jones wrote:
> ---
>  info/Makefile.am                      |   4 +
>  info/info-map-base-allocation-json.sh |  52 ++++++++++
>  info/info-map-base-allocation.sh      |  49 ++++++++++
>  info/nbdinfo.c                        | 134 ++++++++++++++++++++++++--
>  info/nbdinfo.pod                      |  39 +++++++-
>  5 files changed, 271 insertions(+), 7 deletions(-)
An interesting followup might be adding options for --start and --length
to map out only a subset of the file.  That was recently added to
qemu-img map, because there are cases where a file is fragmented enough
that getting the entire map can be time-consuming, in relation to
mapping out just a subset of the image.
> +++ b/info/info-map-base-allocation-json.sh
> @@ -0,0 +1,52 @@
> +#!/usr/bin/env bash
> +# nbd client library in userspace
> +# Copyright (C) 2020 Red Hat Inc.
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
> +
> +. ../tests/functions.sh
> +
> +set -e
> +set -x
> +
> +requires nbdkit --version
> +requires nbdsh --version
Why are we requiring nbdsh in the libnbd testsuite? :)  Is there a
configure option where you can have nbdinfo but not nbdsh?

On the other hand, we _do_ require URI support (libxml2); maybe it's
worth porting 'requires_nbdsh_uri' from nbdkit back to libnbd's
functions.sh, and using that...
> +requires jq --version
> +
> +out=info-base-allocation-json.out
> +cleanup_fn rm -f $out
> +rm -f $out
> +
> +# Note the memory plugin uses a 32K page size, and extents
> +# are always aligned with this.
> +nbdkit -U - memory 1M --run '
> +    nbdsh -u "$uri" \
...since you are depending on it.

> +/* Callback handling --map. */
> +static const char *
> +extent_description (const char *metacontext, uint32_t type)
> +{
> +  if (strcmp (metacontext, "base:allocation") == 0) {
Should we favor the use of LIBNBD_CONTEXT_BASE_ALLOCATION instead of
open-coding this?
> +    switch (type) {
> +    case 0: return "allocated";
> +    case 1: return "zero";
> +    case 2: return "hole";
> +    case 3: return "hole,zero";
> +    }
> +  }
> +  else if (strcmp (metacontext, "qemu:dirty-bitmap") == 0) {
I see you've figured out since that this is not the context name
actually returned by qemu.
> +    switch (type) {
> +    case 0: return "clean";
> +    case 1: return "dirty";
> +    }
> +  }
> +
> +  return "unknown";
> +}
> +
> +++ b/info/nbdinfo.pod
> @@ -4,7 +4,7 @@ nbdinfo - display information and metadata about NBD
servers and exports
>  
>  =head1 SYNOPSIS
>  
> - nbdinfo [--json] [--size] NBD-URI
> + nbdinfo [--json] [--map] [--size] NBD-URI
>  
>   nbdinfo -L|--list NBD-URI
>  
> @@ -20,6 +20,8 @@ nbdinfo - display information and metadata about NBD
servers and exports
>  
>   nbdinfo --size nbd://example.com
>  
> + nbdinfo --map nbd://example.com
> +
>   nbdinfo --json nbd://example.com
>  
>   nbdinfo --list nbd://example.com
> @@ -84,6 +86,32 @@ the I<--json> parameter:
>     ]
>   }
>  
> +=head3 Map
> +
> +To show a map which areas of the disk are allocated and sparse, use
> +the I<--map> option:
> +
> + $ nbdinfo --map nbd://localhost/
> + 0        1048576  0  allocated
> + 1048576  1048576  3  hole,zero
> +
> +The fields are: start size type description.
> +
> +The type field is an integer showing the raw value from the NBD
> +protocol.  For some maps nbdinfo knows how to translate the type into
> +a printable description.
> +
> +By default this shows the C<"base:allocation"> map, but
you can show
> +other maps too:
> +
> + $ nbdinfo --map=qemu:dirty-bitmap nbd://localhost/
> + 0  1048576  1  dirty
> +
> +For more information on NBD maps, see I<Metadata querying> in the
NBD
> +protocol.  I<--json> can also be used here for parsable JSON output.
> +
> +=head3 List of exports
> +
>  To list all the exports available on an NBD server use the I<--list>
>  (I<-L>) option.
Looks like I have more to add here once nbd_opt_info_meta_context lands
for displaying available contexts during --list, and auto-selecting
contexts with --map.

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org