This allows to overlay bad sectors according to the mapfile generated by
ddrescue, to then see where sectors are used using fsck and trying to
copy files around.
Signed-off-by: Fran?ois Revol <revol at free.fr>
---
configure.ac | 2 +
filters/ddrescue/Makefile.am | 75 +++++++
filters/ddrescue/ddrescue.c | 218 ++++++++++++++++++++
filters/ddrescue/nbdkit-ddrescue-filter.pod | 76 +++++++
4 files changed, 371 insertions(+)
create mode 100644 filters/ddrescue/Makefile.am
create mode 100644 filters/ddrescue/ddrescue.c
create mode 100644 filters/ddrescue/nbdkit-ddrescue-filter.pod
diff --git a/configure.ac b/configure.ac
index 6c25226e..21e1013f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -97,6 +97,7 @@ filters="\
cache \
cacheextents \
cow \
+ ddrescue \
delay \
error \
exitlast \
@@ -1089,6 +1090,7 @@ AC_CONFIG_FILES([Makefile
filters/cache/Makefile
filters/cacheextents/Makefile
filters/cow/Makefile
+ filters/ddrescue/Makefile
filters/delay/Makefile
filters/error/Makefile
filters/exitlast/Makefile
diff --git a/filters/ddrescue/Makefile.am b/filters/ddrescue/Makefile.am
new file mode 100644
index 00000000..2498074c
--- /dev/null
+++ b/filters/ddrescue/Makefile.am
@@ -0,0 +1,75 @@
+# nbdkit
+# Copyright (C) 2018 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS
IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+include $(top_srcdir)/common-rules.mk
+
+EXTRA_DIST = \
+ nbdkit-ddrescue-filter.pod \
+ $(NULL)
+
+filter_LTLIBRARIES = nbdkit-ddrescue-filter.la
+
+nbdkit_ddrescue_filter_la_SOURCES = \
+ ddrescue.c \
+ $(top_srcdir)/include/nbdkit-filter.h \
+ $(NULL)
+
+nbdkit_ddrescue_filter_la_CPPFLAGS = \
+ -I$(top_srcdir)/include \
+ -I$(top_srcdir)/common/include \
+ -I$(top_srcdir)/common/sparse \
+ -I$(top_srcdir)/common/utils \
+ $(NULL)
+nbdkit_ddrescue_filter_la_CFLAGS = \
+ $(WARNINGS_CFLAGS) \
+ $(GNUTLS_CFLAGS) \
+ $(NULL)
+nbdkit_ddrescue_filter_la_LDFLAGS = \
+ -module -avoid-version -shared \
+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \
+ $(NULL)
+nbdkit_ddrescue_filter_la_LIBADD = \
+ $(top_builddir)/common/sparse/libsparse.la \
+ $(top_builddir)/common/utils/libutils.la \
+ $(GNUTLS_LIBS) \
+ $(NULL)
+
+if HAVE_POD
+
+man_MANS = nbdkit-ddrescue-filter.1
+CLEANFILES += $(man_MANS)
+
+nbdkit-ddrescue-filter.1: nbdkit-ddrescue-filter.pod
+ $(PODWRAPPER) --section=1 --man $@ \
+ --html $(top_builddir)/html/$@.html \
+ $<
+
+endif HAVE_POD
diff --git a/filters/ddrescue/ddrescue.c b/filters/ddrescue/ddrescue.c
new file mode 100644
index 00000000..a0e49e3c
--- /dev/null
+++ b/filters/ddrescue/ddrescue.c
@@ -0,0 +1,218 @@
+/* nbdkit
+ * Copyright (C) 2018-2020 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS
IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <string.h>
+
+#include <pthread.h>
+
+#include <nbdkit-filter.h>
+
+#include "cleanup.h"
+
+struct range {
+ int64_t start;
+ int64_t end;
+ int64_t size;
+ char status;
+};
+
+struct mapfile {
+ int ranges_count;
+ struct range *ranges;
+};
+
+static struct mapfile map = { 0, NULL };
+
+static int
+parse_mapfile (const char *filename)
+{
+ FILE *fp = NULL;
+ CLEANUP_FREE char *line = NULL;
+ size_t linelen = 0;
+ ssize_t len;
+ int ret = -1;
+ int status_seen = 0;
+
+ fp = fopen (filename, "r");
+ if (!fp) {
+ nbdkit_error ("%s: ddrescue: fopen: %m", filename);
+ goto out;
+ }
+
+ while ((len = getline (&line, &linelen, fp)) != -1) {
+ const char *delim = " \t";
+ char *sp, *p;
+ int64_t offset, length;
+ char status;
+
+ if (len > 0 && line[len-1] == '\n') {
+ line[len-1] = '\0';
+ len--;
+ }
+
+ if (len > 0 && line[0] == '#')
+ continue;
+
+ if (len > 0 && !status_seen) {
+ /* status line, ignore it for now */
+ status_seen = 1;
+ nbdkit_debug ("%s: skipping status line: '%s'",
filename, line);
+ continue;
+ }
+
+ if (sscanf (line, "%" SCNi64 "\t%" SCNi64
"\t%c", &offset, &length, &status) == 3) {
+ if (offset < 0) {
+ nbdkit_error ("block offset must not be negative");
+ return -1;
+ }
+ if (length < 0) {
+ nbdkit_error ("block length must not be negative");
+ return -1;
+ }
+ if (status == '+') {
+ int i = map.ranges_count++;
+ map.ranges = realloc(map.ranges, map.ranges_count * sizeof(struct
range));
+ if (map.ranges == NULL) {
+ nbdkit_error ("%s: ddrescue: realloc: %m", filename);
+ goto out;
+ }
+ map.ranges[i].start = offset;
+ map.ranges[i].end = offset + length - 1;
+ map.ranges[i].size = length;
+ map.ranges[i].status = status;
+ }
+
+ nbdkit_debug ("%s: range: 0x%" PRIx64 " 0x%" PRIx64
" '%c'", filename, offset, length, status);
+ }
+ }
+
+ ret = 0;
+
+ out:
+ if (fp)
+ fclose (fp);
+ return ret;
+}
+
+
+static void
+ddrescue_load (void)
+{
+}
+
+/* On unload, free the sparse array. */
+static void
+ddrescue_unload (void)
+{
+ free (map.ranges);
+ map.ranges = NULL;
+ map.ranges_count = 0;
+}
+
+static int
+ddrescue_config (nbdkit_next_config *next, void *nxdata,
+ const char *key, const char *value)
+{
+ if (strcmp (key, "ddrescue-mapfile") == 0) {
+ if (parse_mapfile (value) == -1)
+ return -1;
+ return 0;
+ }
+
+ else
+ return next (nxdata, key, value);
+}
+
+#define ddrescue_config_help \
+ "ddrescue-mapfile=... Specify ddrescue mapfile to use"
+
+/* We need this because otherwise the layer below can_write is called
+ * and that might return true (eg. if the plugin has a pwrite method
+ * at all), resulting in writes being passed through to the layer
+ * below. This is possibly a bug in nbdkit.
+ */
+static int
+ddrescue_can_write (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle)
+{
+ return 0;
+}
+
+static int
+ddrescue_can_cache (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle)
+{
+ return 0;
+}
+
+/* Read data. */
+static int
+ddrescue_pread (struct nbdkit_next_ops *next_ops, void *nxdata,
+ void *handle, void *buf, uint32_t count, uint64_t offset,
+ uint32_t flags, int *err)
+{
+ int i;
+
+ for (i = 0; i < map.ranges_count; i++) {
+ if (map.ranges[i].status != '+')
+ continue;
+ if (offset >= map.ranges[i].start && offset <=
map.ranges[i].end) {
+ if (offset + count - 1 <= map.ranges[i].end) {
+ /* entirely contained within this range */
+ return next_ops->pread (nxdata, buf, count, offset, flags, err);
+ }
+ }
+ }
+ /* read was not fully covered */
+ *err = EIO;
+ return -1;
+}
+
+static struct nbdkit_filter filter = {
+ .name = "ddrescue",
+ .longname = "nbdkit ddrescue mapfile filter",
+ .load = ddrescue_load,
+ .unload = ddrescue_unload,
+ .config = ddrescue_config,
+ .config_help = ddrescue_config_help,
+ .can_write = ddrescue_can_write,
+ .can_cache = ddrescue_can_cache,
+ .pread = ddrescue_pread,
+};
+
+NBDKIT_REGISTER_FILTER(filter)
diff --git a/filters/ddrescue/nbdkit-ddrescue-filter.pod
b/filters/ddrescue/nbdkit-ddrescue-filter.pod
new file mode 100644
index 00000000..3d9059bd
--- /dev/null
+++ b/filters/ddrescue/nbdkit-ddrescue-filter.pod
@@ -0,0 +1,76 @@
+=head1 NAME
+
+nbdkit-ddrescue-filter - nbdkit filter for serving from ddrescue dump
+
+=head1 SYNOPSIS
+
+ nbdkit --filter=ddrescue plugin [plugin-args...] ddrescue-mapfile=file.map
+
+ nbdkit --filter=ddrescue file file=file.img ddrescue-mapfile=file.map
[plugin-args...]
+
+=head1 DESCRIPTION
+
+C<nbdkit-ddrescue-filter> is a filter for L<nbdkit(1)> which
overlays
+bad blocks according to a GNU L<ddrescue(1)> mapfile. This is mainly
useful
+for testing disk images recovered with ddrescue, to detect which files
+or filesystem structures are impacted, or attempting fsck on them.
+
+Note that the current implementation is read-only.
+
+=head1 EXAMPLES
+
+=over 4
+
+=item Expose a rescued disk image with detected bad sectors:
+
+ nbdkit --filter=ddrescue file file=disk.img ddrescue-mapfile=disk.map
+
+The above command serves the disk image disk.img and maps the bad
+sectors listed in disk.img so that read attempts on them do not return
+a valid block full of zeroes.
+
+=back
+
+=head1 PARAMETERS
+
+The C<ddrescue-mapfile> parameter must point to a valid GNU ddrescue
+mapfile.
+
+=head1 DATA FORMAT
+
+The file pointed to by the C<ddrescue-mapfile> parameter should
+conform to the format of a GNU L<ddrescue(1)> mapfile.
+
+=head1 FILES
+
+=over 4
+
+=item F<$filterdir/nbdkit-ddrescue-filter.so>
+
+The filter.
+
+Use C<nbdkit --dump-config> to find the location of C<$filterdir>.
+
+=back
+
+=head1 VERSION
+
+C<nbdkit-ddrescue-filter> first appeared in nbdkit 1.21.
+
+=head1 SEE ALSO
+
+L<nbdkit(1)>,
+L<nbdkit-file-plugin(1)>,
+L<nbdkit-filter(3)>,
+L<ddrescue(1)>,
+L<https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html>.
+
+=head1 AUTHORS
+
+Fran?ois Revol
+
+Richard W.M. Jones
+
+=head1 COPYRIGHT
+
+Copyright (C) 2020 Red Hat Inc.
--
2.26.2
Richard W.M. Jones
2020-May-01 17:40 UTC
[Libguestfs] [PATCH] WIP: ddrescue mapfile filter
On Fri, May 01, 2020 at 07:14:32PM +0200, Fran?ois Revol wrote:> This allows to overlay bad sectors according to the mapfile generated by > ddrescue, to then see where sectors are used using fsck and trying to > copy files around. > > Signed-off-by: Fran?ois Revol <revol at free.fr>...> +nbdkit_ddrescue_filter_la_LDFLAGS = \ > + -module -avoid-version -shared \We recently added $(SHARED_LDFLAGS) to the end of this line. Compare filters/ip/Makefile.am to your file.> diff --git a/filters/ddrescue/ddrescue.c b/filters/ddrescue/ddrescue.c > new file mode 100644 > index 00000000..a0e49e3c > --- /dev/null > +++ b/filters/ddrescue/ddrescue.c > @@ -0,0 +1,218 @@ > +/* nbdkit > + * Copyright (C) 2018-2020 Red Hat Inc.In several files and the man page you're assigning copyright over to Red Hat, but it's probably better to copyright them to yourself and/or your organisation.> +struct range { > + int64_t start; > + int64_t end; > + int64_t size; > + char status; > +}; > + > +struct mapfile { > + int ranges_count; > + struct range *ranges; > +}; > > +static struct mapfile map = { 0, NULL };There are actually two generic types in common/ which might help here. There's a vector type for building lists: https://github.com/libguestfs/nbdkit/blob/master/common/utils/vector.h Example usage: https://github.com/libguestfs/nbdkit/blob/master/plugins/split/split.c And there's a regions type (built on top of vector) for building contiguous ranges covering a disk, which has a fast look-up method: https://github.com/libguestfs/nbdkit/blob/master/common/regions/regions.h Of course it's optional to use these but they will probably simplify your code.> + > + > +static void > +ddrescue_load (void) > +{ > +}Double blank line before this function, but also this function is empty so you don't need to include it (even though there is an .unload function - .unload will still be called even if .load is missing).> +/* We need this because otherwise the layer below can_write is called > + * and that might return true (eg. if the plugin has a pwrite method > + * at all), resulting in writes being passed through to the layer > + * below. This is possibly a bug in nbdkit. > + */I think this is working as intended, not a bug. However it's good to comment on why the can_write function is needed. [...]> +=back > + > +=head1 VERSION > + > +C<nbdkit-ddrescue-filter> first appeared in nbdkit 1.21.It'll be 1.22 (next stable version after 1.20).> +=head1 SEE ALSO > + > +L<nbdkit(1)>, > +L<nbdkit-file-plugin(1)>, > +L<nbdkit-filter(3)>, > +L<ddrescue(1)>, > +L<https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html>. > + > +=head1 AUTHORS > + > +Fran?ois Revol > + > +Richard W.M. JonesI guess the author is only you!> +=head1 COPYRIGHT > + > +Copyright (C) 2020 Red Hat Inc.See note about copyright above. It generally looks fine to me. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-top is 'top' for virtual machines. Tiny program with many powerful monitoring features, net stats, disk stats, logging, etc. http://people.redhat.com/~rjones/virt-top