Libguestfs - Jul 2016 - [PATCH v1 INCOMPLETE] tests: Implement script to check documented tool options match actual options.

pod-arg-check.pl is run as part of the tests to ensure the
documentation matches the options that the tool implements and vice
versa.

This commit includes scripts to run the tests and various fixes to the
manual pages to ensure that the tests pass.
---
 cat/Makefile.am         |   5 +-
 cat/test-docs.sh        |  26 ++++++++
 cat/virt-cat.pod        |   4 +-
 cat/virt-ls.pod         |   4 +-
 df/Makefile.am          |   3 +-
 df/test-virt-df-docs.sh |  22 +++++++
 df/virt-df.pod          |  10 ++--
 pod-arg-check.pl        | 155 ++++++++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 218 insertions(+), 11 deletions(-)
 create mode 100755 cat/test-docs.sh
 create mode 100755 df/test-virt-df-docs.sh
 create mode 100755 pod-arg-check.pl

diff --git a/cat/Makefile.am b/cat/Makefile.am
index 1a3482b..a831fa4 100644
--- a/cat/Makefile.am
+++ b/cat/Makefile.am
@@ -201,8 +201,11 @@ stamp-virt-ls.pod: virt-ls.pod
 
 TESTS_ENVIRONMENT = $(top_builddir)/run --test
 
-if ENABLE_APPLIANCE
 TESTS = \
+	test-docs.sh
+
+if ENABLE_APPLIANCE
+TESTS += \
 	test-virt-cat.sh \
 	test-virt-filesystems.sh \
 	test-virt-log.sh \
diff --git a/cat/test-docs.sh b/cat/test-docs.sh
new file mode 100755
index 0000000..dea7362
--- /dev/null
+++ b/cat/test-docs.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2016 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+$srcdir/../pod-arg-check.pl virt-cat.pod virt-cat
+$srcdir/../pod-arg-check.pl virt-filesystems.pod virt-filesystems
+$srcdir/../pod-arg-check.pl virt-log.pod virt-log
+$srcdir/../pod-arg-check.pl virt-ls.pod virt-ls \
+                            -ignore=--checksums,--extra-stat,--time,--uid
diff --git a/cat/virt-cat.pod b/cat/virt-cat.pod
index acd43e0..87b0e13 100644
--- a/cat/virt-cat.pod
+++ b/cat/virt-cat.pod
@@ -126,9 +126,9 @@ security problem with malicious guests (CVE-2010-3851).
 Read key or passphrase parameters from stdin.  The default is
 to try to read passphrases from the user by opening F</dev/tty>.
 
-=item B<-m dev[:mountpoint[:options[:fstype]]]>
+=item B<-m> dev[:mountpoint[:options[:fstype]]]
 
-=item B<--mount dev[:mountpoint[:options[:fstype]]]>
+=item B<--mount> dev[:mountpoint[:options[:fstype]]]
 
 Mount the named partition or logical volume on the given mountpoint.
 
diff --git a/cat/virt-ls.pod b/cat/virt-ls.pod
index b90fb51..4957075 100644
--- a/cat/virt-ls.pod
+++ b/cat/virt-ls.pod
@@ -357,9 +357,9 @@ L</RECURSIVE LONG LISTING> above.
 Read key or passphrase parameters from stdin.  The default is
 to try to read passphrases from the user by opening F</dev/tty>.
 
-=item B<-m dev[:mountpoint[:options[:fstype]]]>
+=item B<-m> dev[:mountpoint[:options[:fstype]]]
 
-=item B<--mount dev[:mountpoint[:options[:fstype]]]>
+=item B<--mount> dev[:mountpoint[:options[:fstype]]]
 
 Mount the named partition or logical volume on the given mountpoint.
 
diff --git a/df/Makefile.am b/df/Makefile.am
index 2cf487d..3919397 100644
--- a/df/Makefile.am
+++ b/df/Makefile.am
@@ -95,7 +95,8 @@ stamp-virt-df.pod: virt-df.pod
 
 TESTS_ENVIRONMENT = $(top_builddir)/run --test
 
-TESTS +TESTS = \
+	test-virt-df-docs.sh
 
 if ENABLE_APPLIANCE
 TESTS += \
diff --git a/df/test-virt-df-docs.sh b/df/test-virt-df-docs.sh
new file mode 100755
index 0000000..aabe8d8
--- /dev/null
+++ b/df/test-virt-df-docs.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2016 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+$srcdir/../pod-arg-check.pl virt-df.pod virt-df
diff --git a/df/virt-df.pod b/df/virt-df.pod
index 6419237..57f9fd0 100644
--- a/df/virt-df.pod
+++ b/df/virt-df.pod
@@ -76,20 +76,20 @@ not do this with unrelated guest disks.>
 
 Display brief help.
 
-=item B<-a> file
+=item B<-a> FILE
 
-=item B<--add> file
+=item B<--add> FILE
 
-Add I<file> which should be a disk image from a virtual machine.  If
+Add C<FILE> which should be a disk image from a virtual machine.  If
 the virtual machine has multiple block devices, you must supply all of
 them with separate I<-a> options.
 
 The format of the disk image is auto-detected.  To override this and
 force a particular format use the I<--format=..> option.
 
-=item B<-a URI>
+=item B<-a> URI
 
-=item B<--add URI>
+=item B<--add> URI
 
 Add a remote disk.  See L<guestfish(1)/ADDING REMOTE STORAGE>.
 
diff --git a/pod-arg-check.pl b/pod-arg-check.pl
new file mode 100755
index 0000000..3e9680b
--- /dev/null
+++ b/pod-arg-check.pl
@@ -0,0 +1,155 @@
+#!/usr/bin/env perl
+# pod-arg-check.pl
+# Copyright (C) 2016 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use warnings;
+use strict;
+
+use Pod::Usage;
+use Getopt::Long;
+use Pod::Man;
+
+=head1 NAME
+
+pod-arg-check.pl - Compare man page and tools to check all arguments are
documented
+
+=head1 SYNOPSIS
+
+ pod-arg-check.pl virt-foo.pod ./virt-foo [--ignore=--arg,--arg,...]
+
+=head1 DESCRIPTION
+
+This script compares a manual page (eg. C<virt-foo.pod>) and the
+corresponding tool (eg. C<./virt-foo>) and checks that each command
+line argument is documented in the manual, and that there is no rogue
+documentation for arguments which do not exist.  It works by running
+the tool with the standard C<--long-options> and C<--short-options>
+parameters and comparing their output with the man page.
+
+You can also ignore options, in case this script gets things wrong or
+if there are options that you don't intend to document.
+
+=head1 OPTIONS
+
+=over 4
+
+=cut
+
+my $help;
+
+=item B<--help>
+
+Display brief help.
+
+=cut
+
+my $ignore = "";
+
+=item B<--ignore=--arg,--arg,...>
+
+Ignore the comma-separated list of arguments given.
+
+=cut
+
+# Clean up the program name.
+my $progname = $0;
+$progname =~ s{.*/}{};
+
+# Parse options.
+GetOptions ("help|?" => \$help,
+            "ignore=s" => \$ignore,
+    ) or pod2usage (2);
+pod2usage (1) if $help;
+
+die "$progname: missing argument: pod-arg-check.pl input.pod tool\n"
+    unless @ARGV == 2;
+my $pod = $ARGV[0];
+my $tool = $ARGV[1];
+
+my %ignore = ();
+$ignore{$_} = 1 foreach (split /,/, $ignore);
+
+# Open the man page and slurp it in.
+my $pod_content = "";
+open MAN, $pod or die "$progname: $pod: $!";
+{
+    local $/ = undef;
+    $pod_content = <MAN>;
+}
+close MAN;
+
+# Run the tool with --long-options and --short-options.
+my @tool_options = ();
+open PIPE, "$tool --long-options |"
+    or die "$progname: $tool --long-options: $!";
+while (<PIPE>) {
+    chomp;
+    push @tool_options, $_;
+}
+close PIPE;
+open PIPE, "$tool --short-options |"
+    or die "$progname: $tool --short-options: $!";
+while (<PIPE>) {
+    chomp;
+    push @tool_options, $_;
+}
+close PIPE;
+
+my %tool_option_exists = ();
+$tool_option_exists{$_} = 1 foreach @tool_options;
+
+# Check each option exists in the manual.
+my $tool_options_checked = 0;
+
+foreach (sort keys %tool_option_exists) {
+    unless ($ignore{$_}) {
+        $tool_options_checked++;
+        die "$progname: $pod does not define $_\n"
+            unless $pod_content =~ /^=item B<$_>/m;
+    }
+}
+
+# Check there are no extra options defined in the manual which
+# don't exist in the tool.
+my $pod_options_checked = 0;
+
+my %pod_options = ();
+$pod_options{$_} = 1 foreach ( $pod_content =~ /^=item B<(-[-\w]+)>/gm );
+foreach (sort keys %pod_options) {
+    unless ($ignore{$_}) {
+        $pod_options_checked++;
+        die "$progname: $pod defines option $_ which does not exist in the
tool\n"
+            unless exists $tool_option_exists{$_};
+    }
+}
+
+printf "$progname: $tool: checked $tool_options_checked tool options
against $pod_options_checked documented options\n";
+
+exit 0;
+
+=head1 SEE ALSO
+
+L<podwrapper(1)>,
+libguestfs.git/README.
+
+=head1 AUTHOR
+
+Richard W.M. Jones.
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016 Red Hat Inc.
-- 
2.7.4