Libguestfs - Jun 2016 - [PATCH v8 2/3] New API: filesystem_walk

v8:

  - rebase on master
  - bump version to 1.33.37
  - squash commits 1, 2, 3

Kept original commits messages when squashing them.

Matteo Cafasso (3):
  New API: internal_filesystem_walk
  New API: filesystem_walk
  lib: Added filesystem_walk command tests

 daemon/Makefile.am                |   4 +-
 daemon/tsk.c                      | 249 ++++++++++++++++++++++++++++++++++++++
 docs/guestfs-building.pod         |   4 +
 generator/actions.ml              | 117 ++++++++++++++++++
 generator/structs.ml              |  13 ++
 m4/guestfs_daemon.m4              |   8 ++
 src/MAX_PROC_NR                   |   2 +-
 src/Makefile.am                   |   1 +
 src/tsk.c                         | 129 ++++++++++++++++++++
 tests/tsk/Makefile.am             |   3 +-
 tests/tsk/test-filesystem-walk.sh |  64 ++++++++++
 11 files changed, 591 insertions(+), 3 deletions(-)
 create mode 100644 daemon/tsk.c
 create mode 100644 src/tsk.c
 create mode 100755 tests/tsk/test-filesystem-walk.sh

--
2.8.1

- generator: Added tsk_dirent struct

The tsk_dirent struct contains the information gathered via TSK APIs.

The struct contains the following fields:
 * tsk_inode: inode of a file
 * tsk_type: type of file such as for dirwalk command
 * tsk_size: file size in bytes
 * tsk_name: path relative to its disk partition
 * tsk_flags: bitfield containing extra information

 - configure: Added libtsk compile-time check

Ensure libtsk is available at compile time.
If not, daemon routines depending on it won't be available.

 - API: internal_filesystem_walk

The internal_filesystem_walk command walks through the FS structures
of a disk partition and returns all the files or directories
which could be found.

The command is able to retrieve information regarding deleted
or unaccessible files where other commands such as stat or find
would fail.

The gathered list of tsk_dirent structs is serialised into XDR format
and written to a file by the appliance.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 daemon/Makefile.am        |   4 +-
 daemon/tsk.c              | 249 ++++++++++++++++++++++++++++++++++++++++++++++
 docs/guestfs-building.pod |   4 +
 generator/actions.ml      |   9 ++
 generator/structs.ml      |  13 +++
 m4/guestfs_daemon.m4      |   8 ++
 src/MAX_PROC_NR           |   2 +-
 7 files changed, 287 insertions(+), 2 deletions(-)
 create mode 100644 daemon/tsk.c

diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index fe155e5..b77d1e7 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -179,6 +179,7 @@ guestfsd_SOURCES = \
 	sync.c \
 	syslinux.c \
 	tar.c \
 	truncate.c \
+	tsk.c \
 	umask.c \
 	upload.c \
@@ -209,7 +210,8 @@ guestfsd_LDADD = \
 	$(LIB_CLOCK_GETTIME) \
 	$(LIBINTL) \
 	$(SERVENT_LIB) \
-	$(PCRE_LIBS)
+	$(PCRE_LIBS) \
+	$(TSK_LIBS)

 guestfsd_CPPFLAGS = \
 	-I$(top_srcdir)/gnulib/lib \
diff --git a/daemon/tsk.c b/daemon/tsk.c
new file mode 100644
index 0000000..7ca6ef6
--- /dev/null
+++ b/daemon/tsk.c
@@ -0,0 +1,249 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <string.h>
+#include <unistd.h>
+#include <rpc/xdr.h>
+#include <rpc/types.h>
+
+#include "guestfs_protocol.h"
+#include "daemon.h"
+#include "actions.h"
+#include "optgroups.h"
+
+#ifdef HAVE_LIBTSK
+
+#include <tsk/libtsk.h>
+
+enum tsk_dirent_flags {
+  DIRENT_UNALLOC = 0x00,
+  DIRENT_ALLOC = 0x01,
+  DIRENT_REALLOC = 0x02
+};
+
+static int open_filesystem (const char *, TSK_IMG_INFO **, TSK_FS_INFO **);
+static TSK_WALK_RET_ENUM fswalk_callback (TSK_FS_FILE *, const char *, void *);
+static char file_type (TSK_FS_FILE *);
+static int file_flags (TSK_FS_FILE *fsfile);
+static int send_dirent_info (guestfs_int_tsk_dirent *);
+static void reply_with_tsk_error (const char *);
+
+int
+do_internal_filesystem_walk (const mountable_t *mountable)
+{
+  int ret = -1;
+  TSK_FS_INFO *fs = NULL;
+  TSK_IMG_INFO *img = NULL;  /* Used internally by tsk_fs_dir_walk */
+  int flags = TSK_FS_DIR_WALK_FLAG_ALLOC | TSK_FS_DIR_WALK_FLAG_UNALLOC |
+    TSK_FS_DIR_WALK_FLAG_RECURSE | TSK_FS_DIR_WALK_FLAG_NOORPHAN;
+
+  ret = open_filesystem (mountable->device, &img, &fs);
+  if (ret < 0)
+    return ret;
+
+  reply (NULL, NULL);  /* Reply message. */
+
+  ret = tsk_fs_dir_walk (fs, fs->root_inum, flags, fswalk_callback, NULL);
+  if (ret == 0)
+    ret = send_file_end (0);  /* File transfer end. */
+  else
+    send_file_end (1);  /* Cancel file transfer. */
+
+  fs->close (fs);
+  img->close (img);
+
+  return ret;
+}
+
+/* Inspect the device and initialises the img and fs structures.
+ * Return 0 on success, -1 on error.
+ */
+static int
+open_filesystem (const char *device, TSK_IMG_INFO **img, TSK_FS_INFO **fs)
+{
+  const char *images[] = { device };
+
+  *img = tsk_img_open (1, images, TSK_IMG_TYPE_DETECT, 0);
+  if (*img == NULL) {
+    reply_with_tsk_error ("tsk_image_open");
+    return -1;
+  }
+
+  *fs = tsk_fs_open_img (*img, 0, TSK_FS_TYPE_DETECT);
+  if (*fs == NULL) {
+    reply_with_tsk_error ("tsk_fs_open_img");
+    (*img)->close (*img);
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Filesystem walk callback, it gets called on every FS node.
+ * Parse the node, encode it into an XDR structure and send it to the
appliance.
+ * Return TSK_WALK_CONT on success, TSK_WALK_ERROR on error.
+ */
+static TSK_WALK_RET_ENUM
+fswalk_callback (TSK_FS_FILE *fsfile, const char *path, void *data)
+{
+  int ret = 0;
+  CLEANUP_FREE char *fname = NULL;
+  struct guestfs_int_tsk_dirent dirent;
+
+  /* Ignore ./ and ../ */
+  ret = TSK_FS_ISDOT (fsfile->name->name);
+  if (ret != 0)
+    return TSK_WALK_CONT;
+
+  /* Build the full relative path of the entry */
+  ret = asprintf (&fname, "%s%s", path,
fsfile->name->name);
+  if (ret < 0) {
+    perror ("asprintf");
+    return TSK_WALK_ERROR;
+  }
+
+  dirent.tsk_inode = fsfile->name->meta_addr;
+  dirent.tsk_type = file_type (fsfile);
+  dirent.tsk_size = (fsfile->meta != NULL) ? fsfile->meta->size : -1;
+  dirent.tsk_name = fname;
+  dirent.tsk_flags = file_flags (fsfile);
+
+  ret = send_dirent_info (&dirent);
+  ret = (ret == 0) ? TSK_WALK_CONT : TSK_WALK_ERROR;
+
+  return ret;
+}
+
+/* Inspect fsfile to identify its type. */
+static char
+file_type (TSK_FS_FILE *fsfile)
+{
+  if (fsfile->name->type < TSK_FS_NAME_TYPE_STR_MAX)
+    switch (fsfile->name->type) {
+    case TSK_FS_NAME_TYPE_UNDEF: return 'u';
+    case TSK_FS_NAME_TYPE_FIFO: return 'f';
+    case TSK_FS_NAME_TYPE_CHR: return 'c';
+    case TSK_FS_NAME_TYPE_DIR: return 'd';
+    case TSK_FS_NAME_TYPE_BLK: return 'b';
+    case TSK_FS_NAME_TYPE_REG: return 'r';
+    case TSK_FS_NAME_TYPE_LNK: return 'l';
+    case TSK_FS_NAME_TYPE_SOCK: return 's';
+    case TSK_FS_NAME_TYPE_SHAD: return 'h';
+    case TSK_FS_NAME_TYPE_WHT: return 'w';
+    case TSK_FS_NAME_TYPE_VIRT: return 'u';  /* Temp files created by
TSK */
+    }
+  else if (fsfile->meta != NULL &&
+           fsfile->meta->type < TSK_FS_META_TYPE_STR_MAX)
+    switch (fsfile->name->type) {
+    case TSK_FS_NAME_TYPE_UNDEF: return 'u';
+    case TSK_FS_META_TYPE_REG: return 'r';
+    case TSK_FS_META_TYPE_DIR: return 'd';
+    case TSK_FS_META_TYPE_FIFO: return 'f';
+    case TSK_FS_META_TYPE_CHR: return 'c';
+    case TSK_FS_META_TYPE_BLK: return 'b';
+    case TSK_FS_META_TYPE_LNK: return 'l';
+    case TSK_FS_META_TYPE_SHAD: return 'h';
+    case TSK_FS_META_TYPE_SOCK: return 's';
+    case TSK_FS_META_TYPE_WHT: return 'w';
+    case TSK_FS_META_TYPE_VIRT: return 'u';  /* Temp files created by
TSK */
+    }
+
+  return 'u';
+}
+
+/* Inspect fsfile to retrieve the file allocation state. */
+static int
+file_flags (TSK_FS_FILE *fsfile)
+{
+  int flags = DIRENT_UNALLOC;
+
+  if (fsfile->name->flags & TSK_FS_NAME_FLAG_UNALLOC) {
+    if (fsfile->meta && fsfile->meta->flags &
TSK_FS_META_FLAG_ALLOC)
+      flags |= DIRENT_REALLOC;
+  }
+  else
+    flags |= DIRENT_ALLOC;
+
+  return flags;
+}
+
+/* Serialise dirent into XDR stream and send it to the appliance.
+ * Return 0 on success, -1 on error.
+ */
+static int
+send_dirent_info (guestfs_int_tsk_dirent *dirent)
+{
+  XDR xdr;
+  size_t len = 0;
+  bool_t ret = FALSE;
+  CLEANUP_FREE char *buf = NULL;
+
+  buf = malloc (GUESTFS_MAX_CHUNK_SIZE);
+  if (buf == NULL) {
+    perror ("malloc");
+    return -1;
+  }
+
+  /* Serialise tsk_dirent struct. */
+  xdrmem_create (&xdr, buf, GUESTFS_MAX_CHUNK_SIZE, XDR_ENCODE);
+
+  ret = xdr_guestfs_int_tsk_dirent (&xdr, dirent);
+  if (ret == FALSE) {
+    perror ("xdr_guestfs_int_tsk_dirent");
+    return -1;
+  }
+  len = xdr_getpos (&xdr);
+
+  xdr_destroy (&xdr);
+
+  /* Send serialised tsk_dirent out. */
+  return send_file_write (buf, len);
+}
+
+/* Parse TSK error and send it to the appliance. */
+static void
+reply_with_tsk_error (const char *funcname)
+{
+  int ret = 0;
+  const char *buf = NULL;
+
+  ret = tsk_error_get_errno ();
+  if (ret != 0) {
+    buf = tsk_error_get ();
+    reply_with_error ("%s: %s", funcname, buf);
+  }
+  else
+    reply_with_error ("%s: unknown error", funcname);
+}
+
+int
+optgroup_libtsk_available (void)
+{
+  return 1;
+}
+
+#else   /* !HAVE_LIBTSK */
+
+OPTGROUP_LIBTSK_NOT_AVAILABLE
+
+#endif  /* !HAVE_LIBTSK */
diff --git a/docs/guestfs-building.pod b/docs/guestfs-building.pod
index 28256c4..3fe1d41 100644
--- a/docs/guestfs-building.pod
+++ b/docs/guestfs-building.pod
@@ -395,6 +395,10 @@ Optional.

 Optional.  For tab-completion of commands in bash.

+=item libtsk
+
+Optional.  Library for filesystem forensics analysis.
+
 =back

 =head1 BUILDING FROM GIT
diff --git a/generator/actions.ml b/generator/actions.ml
index a40dbdc..10a6043 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -12989,6 +12989,15 @@ Show all the devices where the filesystems in
C<device> is spanned over.
 If not all the devices for the filesystems are present, then this function
 fails and the C<errno> is set to C<ENODEV>." };

+  { defaults with
+    name = "internal_filesystem_walk"; added = (1, 33, 37);
+    style = RErr, [Mountable "device"; FileOut "filename"],
[];
+    proc_nr = Some 466;
+    visibility = VInternal;
+    optional = Some "libtsk";
+    shortdesc = "walk through the filesystem content";
+    longdesc = "Internal function for filesystem_walk." };
+
 ]

 (* Non-API meta-commands available only in guestfish.
diff --git a/generator/structs.ml b/generator/structs.ml
index 6017ba6..3c2cc61 100644
--- a/generator/structs.ml
+++ b/generator/structs.ml
@@ -444,6 +444,19 @@ let structs = [
     ];
     s_camel_name = "InternalMountable";
   };
+
+  (* The Sleuth Kit directory entry information. *)
+  { defaults with
+    s_name = "tsk_dirent";
+    s_cols = [
+    "tsk_inode", FUInt64;
+    "tsk_type", FChar;
+    "tsk_size", FInt64;
+    "tsk_name", FString;
+    "tsk_flags", FUInt32;
+    ];
+    s_camel_name = "TSKDirent" };
+
 ] (* end of structs *)

 let lookup_struct name diff --git a/m4/guestfs_daemon.m4 b/m4/guestfs_daemon.m4
index 88936b2..192583b 100644
--- a/m4/guestfs_daemon.m4
+++ b/m4/guestfs_daemon.m4
@@ -118,3 +118,11 @@ PKG_CHECK_MODULES([SD_JOURNAL], [libsystemd],[
         AC_MSG_WARN([systemd journal library not found, some features will be
disabled])
     ])
 ])
+
+dnl libtsk sleuthkit library (optional)
+AC_CHECK_LIB([tsk],[tsk_version_print],[
+    AC_CHECK_HEADER([tsk/libtsk.h],[
+        AC_SUBST([TSK_LIBS], [-ltsk])
+        AC_DEFINE([HAVE_LIBTSK], [1], [Define to 1 if The Sleuth Kit library
(libtsk) is available.])
+    ], [])
+],[AC_MSG_WARN([The Sleuth Kit library (libtsk) not found])])
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 073c57b..f27d46f 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-465
+466
--
2.8.1

Library's counterpart of the daemon's internal_filesystem_walk command.

It writes the daemon's command output on a temporary file and parses it,
deserialising the XDR formatted tsk_dirent structs.

It returns to the caller the list of tsk_dirent structs generated by the
internal_filesystem_walk command.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 generator/actions.ml | 108 ++++++++++++++++++++++++++++++++++++++++++
 src/Makefile.am      |   1 +
 src/tsk.c            | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 238 insertions(+)
 create mode 100644 src/tsk.c

diff --git a/generator/actions.ml b/generator/actions.ml
index 10a6043..7edd974 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -3551,6 +3551,114 @@ The environment variable C<XDG_RUNTIME_DIR>
controls the default
 value: If C<XDG_RUNTIME_DIR> is set, then that is the default.
 Else F</tmp> is the default." };

+  { defaults with
+    name = "filesystem_walk"; added = (1, 33, 37);
+    style = RStructList ("dirents", "tsk_dirent"),
[Mountable "device";], [];
+    optional = Some "libtsk";
+    progress = true; cancellable = true;
+    shortdesc = "walk through the filesystem content";
+    longdesc = "\
+Walk through the internal structures of a disk partition
+(eg. F</dev/sda1>) in order to return a list of all the files
+and directories stored within.
+
+It is not necessary to mount the disk partition to run this command.
+
+All entries in the filesystem are returned, excluding C<.> and
+C<..>. This function can list deleted or unaccessible files.
+The entries are I<not> sorted.
+
+The C<tsk_dirent> structure contains the following fields.
+
+=over 4
+
+=item 'tsk_inode'
+
+Filesystem reference number of the node. It migh be C<0>
+if the node has been deleted.
+
+=item 'tsk_type'
+
+Basic file type information.
+See below for a detailed list of values.
+
+=item 'tsk_size'
+
+File size in bytes. It migh be C<-1>
+if the node has been deleted.
+
+=item 'tsk_name'
+
+The file path relative to its directory.
+
+=item 'tsk_flags'
+
+Bitfield containing extra information regarding the entry.
+
+The first bit controls the allocation state of the entry.
+If set to C<1>, the file is allocated and visible within the filesystem.
+Otherwise, the file has been deleted.
+Under certain circumstances, the function C<download_inode>
+can be used to recover deleted files.
+
+The second bit reports whether the metadata structure of the file
+has been reallocated.
+Filesystem such as NTFS and Ext2 or greater, separate the file name
+from the metadata structure.
+The bit is set to C<1> when the file name is in an unallocated state
+and the metadata structure is in an allocated one.
+This generally implies the metadata has been reallocated to a new file.
+Therefore, information such as file type and file size
+might not correspond with the ones of the original deleted entry.
+
+=back
+
+The C<tsk_type> field will contain one of the following characters:
+
+=over 4
+
+=item 'b'
+
+Block special
+
+=item 'c'
+
+Char special
+
+=item 'd'
+
+Directory
+
+=item 'f'
+
+FIFO (named pipe)
+
+=item 'l'
+
+Symbolic link
+
+=item 'r'
+
+Regular file
+
+=item 's'
+
+Socket
+
+=item 'h'
+
+Shadow inode (Solaris)
+
+=item 'w'
+
+Whiteout inode (BSD)
+
+=item 'u'
+
+Unknown file type
+
+=back" };
+
 ]

 (* daemon_functions are any functions which cause some action
diff --git a/src/Makefile.am b/src/Makefile.am
index d659f8d..29586f4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -131,6 +131,7 @@ libguestfs_la_SOURCES = \
 	structs-copy.c \
 	structs-free.c \
 	tmpdirs.c \
+	tsk.c \
 	umask.c \
 	wait.c \
 	whole-file.c \
diff --git a/src/tsk.c b/src/tsk.c
new file mode 100644
index 0000000..83f39df
--- /dev/null
+++ b/src/tsk.c
@@ -0,0 +1,129 @@
+/* libguestfs
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <string.h>
+#include <rpc/xdr.h>
+#include <rpc/types.h>
+
+#include "guestfs.h"
+#include "guestfs_protocol.h"
+#include "guestfs-internal.h"
+#include "guestfs-internal-all.h"
+#include "guestfs-internal-actions.h"
+
+static struct guestfs_tsk_dirent_list *parse_filesystem_walk (guestfs_h *, FILE
*);
+static int deserialise_dirent_list (guestfs_h *, FILE *, struct
guestfs_tsk_dirent_list *);
+
+struct guestfs_tsk_dirent_list *
+guestfs_impl_filesystem_walk (guestfs_h *g, const char *mountable)
+{
+  int ret = 0;
+  CLEANUP_FCLOSE FILE *fp = NULL;
+  CLEANUP_UNLINK_FREE char *tmpfile = NULL;
+
+  ret = guestfs_int_lazy_make_tmpdir (g);
+  if (ret < 0)
+    return NULL;
+
+  tmpfile = safe_asprintf (g, "%s/filesystem_walk%d", g->tmpdir,
++g->unique);
+
+  ret = guestfs_internal_filesystem_walk (g, mountable, tmpfile);
+  if (ret < 0)
+    return NULL;
+
+  fp = fopen (tmpfile, "r");
+  if (fp == NULL) {
+    perrorf (g, "fopen: %s", tmpfile);
+    return NULL;
+  }
+
+  return parse_filesystem_walk (g, fp);  /* caller frees */
+}
+
+/* Parse the file content and return dirents list.
+ * Return a list of tsk_dirent on success, NULL on error.
+ */
+static struct guestfs_tsk_dirent_list *
+parse_filesystem_walk (guestfs_h *g, FILE *fp)
+{
+  int ret = 0;
+  struct guestfs_tsk_dirent_list *dirents = NULL;
+
+  /* Initialise results array. */
+  dirents = safe_malloc (g, sizeof (*dirents));
+  dirents->len = 8;
+  dirents->val = safe_malloc (g, dirents->len * sizeof
(*dirents->val));
+
+  /* Deserialise buffer into dirent list. */
+  ret = deserialise_dirent_list (g, fp, dirents);
+  if (ret < 0) {
+    guestfs_free_tsk_dirent_list (dirents);
+    return NULL;
+  }
+
+  return dirents;
+}
+
+/* Deserialise the file content and populate the dirent list.
+ * Return the number of deserialised dirents, -1 on error.
+ */
+static int
+deserialise_dirent_list (guestfs_h *g, FILE *fp,
+                         struct guestfs_tsk_dirent_list *dirents)
+{
+  XDR xdr;
+  bool_t ret = 0;
+  int statret = 0;
+  uint32_t index = 0;
+  struct stat statbuf;
+
+  statret = fstat (fileno(fp), &statbuf);
+  if (statret == -1)
+    return -1;
+
+  xdrstdio_create (&xdr, fp, XDR_DECODE);
+
+  for (index = 0; xdr_getpos (&xdr) < statbuf.st_size; index++) {
+    if (index == dirents->len) {
+      dirents->len = 2 * dirents->len;
+      dirents->val = safe_realloc (g, dirents->val,
+                                   dirents->len *
+                                   sizeof (*dirents->val));
+    }
+
+    /* Clear the entry so xdr logic will allocate necessary memory. */
+    memset (&dirents->val[index], 0, sizeof (*dirents->val));
+    ret = xdr_guestfs_int_tsk_dirent (&xdr, (guestfs_int_tsk_dirent *)
+                                      &dirents->val[index]);
+    if (ret == FALSE)
+      break;
+  }
+
+  xdr_destroy (&xdr);
+  dirents->len = index;
+
+  return (ret == TRUE) ? 0 : -1;
+}
--
2.8.1

The tests check whether the filesystem_walk command is able to retrieve
information regarding both existing and deleted files.

A NTFS image is used as Ext3+ filesystems deletion is more aggressive
in terms of metadata removal.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 tests/tsk/Makefile.am             |  3 +-
 tests/tsk/test-filesystem-walk.sh | 64 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100755 tests/tsk/test-filesystem-walk.sh

diff --git a/tests/tsk/Makefile.am b/tests/tsk/Makefile.am
index 0cd7c03..f9b2fef 100644
--- a/tests/tsk/Makefile.am
+++ b/tests/tsk/Makefile.am
@@ -18,7 +18,8 @@
 include $(top_srcdir)/subdir-rules.mk

 TESTS = \
-	test-download-inode.sh
+	test-download-inode.sh \
+	test-filesystem-walk.sh

 TESTS_ENVIRONMENT = $(top_builddir)/run --test

diff --git a/tests/tsk/test-filesystem-walk.sh
b/tests/tsk/test-filesystem-walk.sh
new file mode 100755
index 0000000..c57f979
--- /dev/null
+++ b/tests/tsk/test-filesystem-walk.sh
@@ -0,0 +1,64 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2016 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test the filesystem-walk command.
+
+if [ -n "$SKIP_TEST_FILESYSTEM_WALK_SH" ]; then
+    echo "$0: test skipped because environment variable is set."
+    exit 77
+fi
+
+# Skip if TSK is not supported by the appliance.
+if ! guestfish add /dev/null : run : available "libtsk"; then
+    echo "$0: skipped because TSK is not available in the appliance"
+    exit 77
+fi
+
+if [ ! -s ../../test-data/phony-guests/windows.img ]; then
+    echo "$0: skipped because windows.img is zero-sized"
+    exit 77
+fi
+
+output=$(
+guestfish --ro -a ../../test-data/phony-guests/windows.img <<EOF
+run
+mount /dev/sda2 /
+write /test.txt "foobar"
+rm /test.txt
+umount /
+filesystem-walk /dev/sda2
+EOF
+)
+
+# test $MFT is in the list
+echo $output | grep -zq "{ tsk_inode: 0 tsk_type: r tsk_size: .* tsk_name:
\$MFT tsk_flags: 1 }"
+if [ $? != 0 ]; then
+    echo "$0: \$MFT not found in files list."
+    echo "File list:"
+    echo $output
+    exit 1
+fi
+
+# test deleted file is in the list
+echo $output | grep -zq "{ tsk_inode: .* tsk_type: [ru] tsk_size: .*
tsk_name: test.txt tsk_flags: 0 }"
+if [ $? != 0 ]; then
+    echo "$0: /test.txt not found in files list."
+    echo "File list:"
+    echo $output
+    exit 1
+fi
--
2.8.1

On Monday 13 June 2016 19:50:52 Matteo Cafasso wrote:
>  - generator: Added tsk_dirent struct
> 
> The tsk_dirent struct contains the information gathered via TSK APIs.
> 
> The struct contains the following fields:
>  * tsk_inode: inode of a file
>  * tsk_type: type of file such as for dirwalk command
>  * tsk_size: file size in bytes
>  * tsk_name: path relative to its disk partition
>  * tsk_flags: bitfield containing extra information
> 
>  - configure: Added libtsk compile-time check
> 
> Ensure libtsk is available at compile time.
> If not, daemon routines depending on it won't be available.
> 
>  - API: internal_filesystem_walk
> 
> The internal_filesystem_walk command walks through the FS structures
> of a disk partition and returns all the files or directories
> which could be found.
> 
> The command is able to retrieve information regarding deleted
> or unaccessible files where other commands such as stat or find
> would fail.
> 
> The gathered list of tsk_dirent structs is serialised into XDR format
> and written to a file by the appliance.
> 
> Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
> ---
Note that a new struct adds new content to the following files:

  gobject/Makefile.inc
  java/Makefile.inc
  java/com/redhat/et/libguestfs/.gitignore

These changes should be part of this patch too (no need to resend the
series just because of that for now, though).

-- 
Pino Toscano

On Mon, Jun 13, 2016 at 07:50:52PM +0300, Matteo Cafasso
wrote:
> diff --git a/generator/structs.ml b/generator/structs.ml
> index 6017ba6..3c2cc61 100644
> --- a/generator/structs.ml
> +++ b/generator/structs.ml
> @@ -444,6 +444,19 @@ let structs = [
>      ];
>      s_camel_name = "InternalMountable";
>    };
> +
> +  (* The Sleuth Kit directory entry information. *)
> +  { defaults with
> +    s_name = "tsk_dirent";
> +    s_cols = [
> +    "tsk_inode", FUInt64;
> +    "tsk_type", FChar;
> +    "tsk_size", FInt64;
> +    "tsk_name", FString;
> +    "tsk_flags", FUInt32;
Note if you ever need to add more columns in future, you won't be able
to, unless you reserve some space in the struct now by adding:

   "tsk_spare1", FInt64;
   "tsk_spare2", FInt64;
   "tsk_spare3", FInt64;
   "tsk_spare4", FInt64;
   "tsk_spare5", FInt64;
   "tsk_spare6", FInt64;

I can't say if you'll need more columns here, or if the set you have
now is the final set that you'll ever need.

This patch looks OK to me.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-top is 'top' for virtual machines.  Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://people.redhat.com/~rjones/virt-top

On Mon, Jun 13, 2016 at 07:50:53PM +0300, Matteo Cafasso
wrote:
> +=item 'tsk_flags'
> +
> +Bitfield containing extra information regarding the entry.
> +
> +The first bit controls the allocation state of the entry.
> +If set to C<1>, the file is allocated and visible within the
filesystem.
> +Otherwise, the file has been deleted.
> +Under certain circumstances, the function C<download_inode>
> +can be used to recover deleted files.
> +
> +The second bit reports whether the metadata structure of the file
> +has been reallocated.
> +Filesystem such as NTFS and Ext2 or greater, separate the file name
> +from the metadata structure.
> +The bit is set to C<1> when the file name is in an unallocated state
> +and the metadata structure is in an allocated one.
> +This generally implies the metadata has been reallocated to a new file.
> +Therefore, information such as file type and file size
> +might not correspond with the ones of the original deleted entry.
Probably better to give actual values rather than "the first bit"
and so on.

eg:

=item 'tsk_flags'

Bitfield containing extra information about this entry.  It
contains the logical OR of the following values:

=over 4

=item 0x0001

If set, the file is allocated and visible within the filesystem
[etc etc]

=item 0x0002

[etc etc]

=back
> +/* Deserialise the file content and populate the dirent list.
> + * Return the number of deserialised dirents, -1 on error.
> + */
> +static int
> +deserialise_dirent_list (guestfs_h *g, FILE *fp,
> +                         struct guestfs_tsk_dirent_list *dirents)
> +{
> +  XDR xdr;
> +  bool_t ret = 0;
I don't know where 'bool_t' comes from, but don't use it.  Put
#include <stdbool.h> at the top of the file, and use the C99
'bool'
type and 'true' and 'false' values.  See:

http://pubs.opengroup.org/onlinepubs/009695399/basedefs/stdbool.h.html
> +  return (ret == TRUE) ? 0 : -1;
You can just write:

  return ret ? 0 : -1;

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v

On Mon, Jun 13, 2016 at 07:50:54PM +0300, Matteo Cafasso
wrote:
> The tests check whether the filesystem_walk command is able to retrieve
> information regarding both existing and deleted files.
> 
> A NTFS image is used as Ext3+ filesystems deletion is more aggressive
> in terms of metadata removal.
> 
> Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
> ---
>  tests/tsk/Makefile.am             |  3 +-
>  tests/tsk/test-filesystem-walk.sh | 64
+++++++++++++++++++++++++++++++++++++++
>  2 files changed, 66 insertions(+), 1 deletion(-)
>  create mode 100755 tests/tsk/test-filesystem-walk.sh
> 
> diff --git a/tests/tsk/Makefile.am b/tests/tsk/Makefile.am
> index 0cd7c03..f9b2fef 100644
> --- a/tests/tsk/Makefile.am
> +++ b/tests/tsk/Makefile.am
> @@ -18,7 +18,8 @@
>  include $(top_srcdir)/subdir-rules.mk
> 
>  TESTS = \
> -	test-download-inode.sh
> +	test-download-inode.sh \
> +	test-filesystem-walk.sh
> 
>  TESTS_ENVIRONMENT = $(top_builddir)/run --test
> 
> diff --git a/tests/tsk/test-filesystem-walk.sh
b/tests/tsk/test-filesystem-walk.sh
> new file mode 100755
> index 0000000..c57f979
> --- /dev/null
> +++ b/tests/tsk/test-filesystem-walk.sh
> @@ -0,0 +1,64 @@
> +#!/bin/bash -
> +# libguestfs
> +# Copyright (C) 2016 Red Hat Inc.
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA.
> +
> +# Test the filesystem-walk command.
> +
> +if [ -n "$SKIP_TEST_FILESYSTEM_WALK_SH" ]; then
> +    echo "$0: test skipped because environment variable is set."
> +    exit 77
> +fi
> +
> +# Skip if TSK is not supported by the appliance.
> +if ! guestfish add /dev/null : run : available "libtsk"; then
> +    echo "$0: skipped because TSK is not available in the
appliance"
> +    exit 77
> +fi
> +
> +if [ ! -s ../../test-data/phony-guests/windows.img ]; then
> +    echo "$0: skipped because windows.img is zero-sized"
> +    exit 77
> +fi
> +
> +output=$(
> +guestfish --ro -a ../../test-data/phony-guests/windows.img <<EOF
> +run
> +mount /dev/sda2 /
> +write /test.txt "foobar"
> +rm /test.txt
> +umount /
> +filesystem-walk /dev/sda2
> +EOF
> +)
> +
> +# test $MFT is in the list
> +echo $output | grep -zq "{ tsk_inode: 0 tsk_type: r tsk_size: .*
tsk_name: \$MFT tsk_flags: 1 }"
> +if [ $? != 0 ]; then
> +    echo "$0: \$MFT not found in files list."
> +    echo "File list:"
> +    echo $output
> +    exit 1
> +fi
> +
> +# test deleted file is in the list
> +echo $output | grep -zq "{ tsk_inode: .* tsk_type: [ru] tsk_size: .*
tsk_name: test.txt tsk_flags: 0 }"
> +if [ $? != 0 ]; then
> +    echo "$0: /test.txt not found in files list."
> +    echo "File list:"
> +    echo $output
> +    exit 1
> +fi
This one looks OK.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into KVM guests.
http://libguestfs.org/virt-v2v