Libguestfs - Sep 2016 - [PATCH 4/4] TSK: small refactoring

This series is ready for review but requires the previous one to be merged
first:
https://www.redhat.com/archives/libguestfs/2016-September/msg00101.html

The find_block API allows the User to search all the filesystem entries
referring to a given data block and returns a tsk_dirent structure
for each of them.

Use case examples:

 - Check whether a block containing a deleted file has been re-used to store a
new one.
 - Map a certain area of a disk with the contained files.

Matteo Cafasso (4):
  New API: internal_find_block
  New API: find_block
  find_block: added API tests
  TSK: small refactoring

 daemon/tsk.c                 | 89 ++++++++++++++++++++++++++++++++++++++++++++
 generator/actions.ml         | 25 +++++++++++++
 src/MAX_PROC_NR              |  2 +-
 src/tsk.c                    | 65 +++++++++++++++++++-------------
 tests/tsk/Makefile.am        |  1 +
 tests/tsk/test-find-block.sh | 66 ++++++++++++++++++++++++++++++++
 6 files changed, 221 insertions(+), 27 deletions(-)
 create mode 100755 tests/tsk/test-find-block.sh

--
2.9.3

The internal_find_block command searches all entries referring to the
given filesystem data block and returns a tsk_dirent structure
for each of them.

For filesystems such as NTFS which do not delete the block mapping
when removing files, it is possible to get multiple non-allocated
entries for the same block.

The gathered list of tsk_dirent structs is serialised into XDR format
and written to a file by the appliance.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 daemon/tsk.c         | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 generator/actions.ml |  9 ++++++
 src/MAX_PROC_NR      |  2 +-
 3 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/daemon/tsk.c b/daemon/tsk.c
index af803d7..6ac3952 100644
--- a/daemon/tsk.c
+++ b/daemon/tsk.c
@@ -42,9 +42,16 @@ enum tsk_dirent_flags {
   DIRENT_COMPRESSED = 0x04
 };

+typedef struct {
+  int found;
+  uint64_t block;
+} findblk_data;
+
 static int open_filesystem (const char *, TSK_IMG_INFO **, TSK_FS_INFO **);
 static TSK_WALK_RET_ENUM fswalk_callback (TSK_FS_FILE *, const char *, void *);
 static TSK_WALK_RET_ENUM findino_callback (TSK_FS_FILE *, const char *, void
*);
+static TSK_WALK_RET_ENUM findblk_callback (TSK_FS_FILE *, const char *, void
*);
+static TSK_WALK_RET_ENUM attrwalk_callback (TSK_FS_FILE *, TSK_OFF_T ,
TSK_DADDR_T , char *, size_t , TSK_FS_BLOCK_FLAG_ENUM , void *);
 static int send_dirent_info (TSK_FS_FILE *, const char *);
 static char file_type (TSK_FS_FILE *);
 static int file_flags (TSK_FS_FILE *fsfile);
@@ -109,6 +116,36 @@ do_internal_find_inode (const mountable_t *mountable,
int64_t inode)
   return ret;
 }

+int
+do_internal_find_block (const mountable_t *mountable, int64_t block)
+{
+  int ret = -1;
+  TSK_FS_INFO *fs = NULL;
+  TSK_IMG_INFO *img = NULL;  /* Used internally by tsk_fs_dir_walk */
+  const int flags +    TSK_FS_DIR_WALK_FLAG_ALLOC |
TSK_FS_DIR_WALK_FLAG_UNALLOC |
+    TSK_FS_DIR_WALK_FLAG_RECURSE | TSK_FS_DIR_WALK_FLAG_NOORPHAN;
+
+  ret = open_filesystem (mountable->device, &img, &fs);
+  if (ret < 0)
+    return ret;
+
+  reply (NULL, NULL);  /* Reply message. */
+
+  ret = tsk_fs_dir_walk (fs, fs->root_inum, flags,
+                         findblk_callback, (void *) &block);
+  if (ret == 0)
+    ret = send_file_end (0);  /* File transfer end. */
+  else
+    send_file_end (1);  /* Cancel file transfer. */
+
+  fs->close (fs);
+  img->close (img);
+
+  return ret;
+}
+
+
 /* Inspect the device and initialises the img and fs structures.
  * Return 0 on success, -1 on error.
  */
@@ -172,6 +209,58 @@ findino_callback (TSK_FS_FILE *fsfile, const char *path,
void *data)
   return (ret == 0) ? TSK_WALK_CONT : TSK_WALK_ERROR;
 }

+/* Find block, it gets called on every FS node.
+ *
+ * Return TSK_WALK_CONT on success, TSK_WALK_ERROR on error.
+ */
+static TSK_WALK_RET_ENUM
+findblk_callback (TSK_FS_FILE *fsfile, const char *path, void *data)
+{
+  findblk_data blkdata;
+  const TSK_FS_ATTR *fsattr = NULL;
+  int ret = 0, count = 0, index = 0;
+  uint64_t *block = (uint64_t *) data;
+  const int flags = TSK_FS_FILE_WALK_FLAG_AONLY | TSK_FS_FILE_WALK_FLAG_SLACK;
+
+  if (entry_is_dot (fsfile))
+    return TSK_WALK_CONT;
+
+  blkdata.found = 0;
+  blkdata.block = *block;
+
+  /* Retrieve block list */
+  count = tsk_fs_file_attr_getsize (fsfile);
+
+  for (index = 0; index < count; index++) {
+    fsattr = tsk_fs_file_attr_get_idx (fsfile, index);
+
+    if (fsattr != NULL && fsattr->flags & TSK_FS_ATTR_NONRES)
+      tsk_fs_attr_walk (fsattr, flags, attrwalk_callback, (void*)
&blkdata);
+  }
+
+  if (blkdata.found)
+    ret = send_dirent_info (fsfile, path);
+
+  return (ret == 0) ? TSK_WALK_CONT : TSK_WALK_ERROR;
+}
+
+/* Attribute walk, searches the given block within the FS node attributes. */
+static TSK_WALK_RET_ENUM
+attrwalk_callback (TSK_FS_FILE *fsfile, TSK_OFF_T offset,
+                   TSK_DADDR_T blkaddr, char *buf, size_t size,
+                   TSK_FS_BLOCK_FLAG_ENUM flags, void *data)
+{
+  findblk_data *blkdata = (findblk_data *) data;
+
+  if (blkaddr == blkdata->block) {
+    blkdata->found = 1;
+
+    return TSK_WALK_STOP;
+  }
+
+  return TSK_WALK_CONT;
+}
+
 /* Extract the information from the entry, serialize and send it out.
  * Return 0 on success, -1 on error.
  */
diff --git a/generator/actions.ml b/generator/actions.ml
index 91a1819..b38a30f 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -13253,6 +13253,15 @@ is removed." };
     shortdesc = "search the entries associated to the given inode";
     longdesc = "Internal function for find_inode." };

+  { defaults with
+    name = "internal_find_block"; added = (1, 35, 6);
+    style = RErr, [Mountable "device"; Int64 "block";
FileOut "filename";], [];
+    proc_nr = Some 471;
+    visibility = VInternal;
+    optional = Some "libtsk";
+    shortdesc = "search the entries associated to the given block";
+    longdesc = "Internal function for find_block." };
+
 ]

 (* Non-API meta-commands available only in guestfish.
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 5f476b6..c305aa5 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-470
+471
--
2.9.3

Library's counterpart of the daemon's internal_find_block command.

It writes the daemon's command output on a temporary file and parses it,
deserialising the XDR formatted tsk_dirent structs.

It returns to the caller the list of tsk_dirent structs generated by the
internal_find_block command.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 generator/actions.ml | 16 ++++++++++++++++
 src/tsk.c            | 26 ++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/generator/actions.ml b/generator/actions.ml
index b38a30f..8947551 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -3729,6 +3729,22 @@ Searches all the entries associated with the given inode.
 For each entry, a C<tsk_dirent> structure is returned.
 See C<filesystem_walk> for more information about C<tsk_dirent>
structures." };

+  { defaults with
+    name = "find_block"; added = (1, 35, 6);
+    style = RStructList ("dirents", "tsk_dirent"),
[Mountable "device"; Int64 "block";], [];
+    optional = Some "libtsk";
+    progress = true; cancellable = true;
+    shortdesc = "search the entries referring to the given data
block";
+    longdesc = "\
+Searches all the entries referring to the given data block.
+
+Certain filesystems preserve the block mapping when deleting a file.
+Therefore, it is possible to see multiple deleted files referring
+to the same block.
+
+For each entry, a C<tsk_dirent> structure is returned.
+See C<filesystem_walk> for more information about C<tsk_dirent>
structures." };
+
 ]

 (* daemon_functions are any functions which cause some action
diff --git a/src/tsk.c b/src/tsk.c
index d3da834..6b1b29c 100644
--- a/src/tsk.c
+++ b/src/tsk.c
@@ -89,6 +89,32 @@ guestfs_impl_find_inode (guestfs_h *g, const char *mountable,
int64_t inode)
   return parse_dirent_file (g, fp);  /* caller frees */
 }

+struct guestfs_tsk_dirent_list *
+guestfs_impl_find_block (guestfs_h *g, const char *mountable, int64_t block)
+{
+  int ret = 0;
+  CLEANUP_FCLOSE FILE *fp = NULL;
+  CLEANUP_UNLINK_FREE char *tmpfile = NULL;
+
+  ret = guestfs_int_lazy_make_tmpdir (g);
+  if (ret < 0)
+    return NULL;
+
+  tmpfile = safe_asprintf (g, "%s/find_block%d", g->tmpdir,
++g->unique);
+
+  ret = guestfs_internal_find_block (g, mountable, block, tmpfile);
+  if (ret < 0)
+    return NULL;
+
+  fp = fopen (tmpfile, "r");
+  if (fp == NULL) {
+    perrorf (g, "fopen: %s", tmpfile);
+    return NULL;
+  }
+
+  return parse_dirent_file (g, fp);  /* caller frees */
+}
+
 /* Parse the file content and return dirents list.
  * Return a list of tsk_dirent on success, NULL on error.
  */
--
2.9.3

NTFS file system always has the Boot file at block 0. This reliable
information helps testing the API.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 tests/tsk/Makefile.am        |  1 +
 tests/tsk/test-find-block.sh | 66 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100755 tests/tsk/test-find-block.sh

diff --git a/tests/tsk/Makefile.am b/tests/tsk/Makefile.am
index 07c74f9..44a893e 100644
--- a/tests/tsk/Makefile.am
+++ b/tests/tsk/Makefile.am
@@ -21,6 +21,7 @@ TESTS = \
 	test-download-inode.sh \
 	test-download-blocks.sh \
 	test-filesystem-walk.sh \
+	test-find-block.sh \
 	test-find-inode.sh

 TESTS_ENVIRONMENT = $(top_builddir)/run --test
diff --git a/tests/tsk/test-find-block.sh b/tests/tsk/test-find-block.sh
new file mode 100755
index 0000000..984947d
--- /dev/null
+++ b/tests/tsk/test-find-block.sh
@@ -0,0 +1,66 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2016 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test the find-block command.
+
+if [ -n "$SKIP_TEST_FIND_BLOCK_SH" ]; then
+    echo "$0: test skipped because environment variable is set."
+    exit 77
+fi
+
+# Skip if TSK is not supported by the appliance.
+if ! guestfish add /dev/null : run : available "libtsk"; then
+    echo "$0: skipped because TSK is not available in the appliance"
+    exit 77
+fi
+
+if [ ! -s ../../test-data/phony-guests/windows.img ]; then
+    echo "$0: skipped because windows.img is zero-sized"
+    exit 77
+fi
+
+output=$(
+guestfish --ro -a ../../test-data/phony-guests/windows.img <<EOF
+run
+find-block /dev/sda2 0
+EOF
+)
+
+# test $Boot is in the list
+echo $output | grep -zq '{ tsk_inode: 7
+tsk_type: r
+tsk_size: .*
+tsk_name: \$Boot
+tsk_flags: 1
+tsk_atime_sec: .*
+tsk_atime_nsec: .*
+tsk_mtime_sec: .*
+tsk_mtime_nsec: .*
+tsk_ctime_sec: .*
+tsk_ctime_nsec: .*
+tsk_crtime_sec: .*
+tsk_crtime_nsec: .*
+tsk_nlink: 1
+tsk_link:
+tsk_spare1: 0 }'
+if [ $? != 0 ]; then
+    echo "$0: \$Boot not found in files list."
+    echo "File list:"
+    echo $output
+    exit 1
+fi
--
2.9.3

Removed duplicated code.

Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
---
 src/tsk.c | 69 ++++++++++++++++++++++++++-------------------------------------
 1 file changed, 28 insertions(+), 41 deletions(-)

diff --git a/src/tsk.c b/src/tsk.c
index 6b1b29c..8e6d266 100644
--- a/src/tsk.c
+++ b/src/tsk.c
@@ -34,96 +34,71 @@
 #include "guestfs-internal-all.h"
 #include "guestfs-internal-actions.h"

-static struct guestfs_tsk_dirent_list *parse_dirent_file (guestfs_h *, FILE *);
+static struct guestfs_tsk_dirent_list *parse_dirent_file (guestfs_h *, const
char *);
 static int deserialise_dirent_list (guestfs_h *, FILE *, struct
guestfs_tsk_dirent_list *);
+static char *make_temp_file (guestfs_h *, const char *);

 struct guestfs_tsk_dirent_list *
 guestfs_impl_filesystem_walk (guestfs_h *g, const char *mountable)
 {
   int ret = 0;
-  CLEANUP_FCLOSE FILE *fp = NULL;
   CLEANUP_UNLINK_FREE char *tmpfile = NULL;

-  ret = guestfs_int_lazy_make_tmpdir (g);
-  if (ret < 0)
-    return NULL;
-
-  tmpfile = safe_asprintf (g, "%s/filesystem_walk%d", g->tmpdir,
++g->unique);
+  tmpfile = make_temp_file (g, "filesystem_walk");

   ret = guestfs_internal_filesystem_walk (g, mountable, tmpfile);
   if (ret < 0)
     return NULL;

-  fp = fopen (tmpfile, "r");
-  if (fp == NULL) {
-    perrorf (g, "fopen: %s", tmpfile);
-    return NULL;
-  }
-
-  return parse_dirent_file (g, fp);  /* caller frees */
+  return parse_dirent_file (g, tmpfile);  /* caller frees */
 }

 struct guestfs_tsk_dirent_list *
 guestfs_impl_find_inode (guestfs_h *g, const char *mountable, int64_t inode)
 {
   int ret = 0;
-  CLEANUP_FCLOSE FILE *fp = NULL;
   CLEANUP_UNLINK_FREE char *tmpfile = NULL;

-  ret = guestfs_int_lazy_make_tmpdir (g);
-  if (ret < 0)
-    return NULL;
-
-  tmpfile = safe_asprintf (g, "%s/find_inode%d", g->tmpdir,
++g->unique);
+  tmpfile = make_temp_file (g, "find_inode");

   ret = guestfs_internal_find_inode (g, mountable, inode, tmpfile);
   if (ret < 0)
     return NULL;

-  fp = fopen (tmpfile, "r");
-  if (fp == NULL) {
-    perrorf (g, "fopen: %s", tmpfile);
-    return NULL;
-  }
-
-  return parse_dirent_file (g, fp);  /* caller frees */
+  return parse_dirent_file (g, tmpfile);  /* caller frees */
 }

 struct guestfs_tsk_dirent_list *
 guestfs_impl_find_block (guestfs_h *g, const char *mountable, int64_t block)
 {
   int ret = 0;
-  CLEANUP_FCLOSE FILE *fp = NULL;
   CLEANUP_UNLINK_FREE char *tmpfile = NULL;

-  ret = guestfs_int_lazy_make_tmpdir (g);
-  if (ret < 0)
-    return NULL;
-
-  tmpfile = safe_asprintf (g, "%s/find_block%d", g->tmpdir,
++g->unique);
+  tmpfile = make_temp_file (g, "find_block");

   ret = guestfs_internal_find_block (g, mountable, block, tmpfile);
   if (ret < 0)
     return NULL;

-  fp = fopen (tmpfile, "r");
-  if (fp == NULL) {
-    perrorf (g, "fopen: %s", tmpfile);
-    return NULL;
-  }
-
-  return parse_dirent_file (g, fp);  /* caller frees */
+  return parse_dirent_file (g, tmpfile);  /* caller frees */
 }

 /* Parse the file content and return dirents list.
  * Return a list of tsk_dirent on success, NULL on error.
  */
 static struct guestfs_tsk_dirent_list *
-parse_dirent_file (guestfs_h *g, FILE *fp)
+parse_dirent_file (guestfs_h *g, const char *tmpfile)
 {
   int ret = 0;
+  CLEANUP_FCLOSE FILE *fp = NULL;
   struct guestfs_tsk_dirent_list *dirents = NULL;

+  fp = fopen (tmpfile, "r");
+  if (fp == NULL) {
+    perrorf (g, "fopen: %s", tmpfile);
+    return NULL;
+  }
+
   /* Initialise results array. */
   dirents = safe_malloc (g, sizeof (*dirents));
   dirents->len = 8;
@@ -178,3 +153,15 @@ deserialise_dirent_list (guestfs_h *g, FILE *fp,

   return ret ? 0 : -1;
 }
+
+static char *
+make_temp_file (guestfs_h *g, const char *name)
+{
+  int ret = 0;
+
+  ret = guestfs_int_lazy_make_tmpdir (g);
+  if (ret < 0)
+    return NULL;
+
+  return safe_asprintf (g, "%s/%s%d", g->tmpdir, name,
++g->unique);
+}
--
2.9.3

On Saturday, 17 September 2016 18:18:53 CEST Matteo Cafasso
wrote:
> The internal_find_block command searches all entries referring to the
> given filesystem data block and returns a tsk_dirent structure
> for each of them.
> 
> For filesystems such as NTFS which do not delete the block mapping
> when removing files, it is possible to get multiple non-allocated
> entries for the same block.
> 
> The gathered list of tsk_dirent structs is serialised into XDR format
> and written to a file by the appliance.
> 
> Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
> ---
LGTM, just one small note below.
> +typedef struct {
> +  int found;
This can be a simple bool (including <stdbool.h> to use it).
> +  uint64_t block;
> +} findblk_data;
Thanks,
-- 
Pino Toscano

On Saturday, 17 September 2016 18:18:56 CEST Matteo Cafasso
wrote:
> Removed duplicated code.
> 
> Signed-off-by: Matteo Cafasso <noxdafox@gmail.com>
> ---
As mentioned in another email, I'd like this to be a single change
with the return parse_filesystem_walk -> parse_dirent_file rename.
>  struct guestfs_tsk_dirent_list *
>  guestfs_impl_filesystem_walk (guestfs_h *g, const char *mountable)
>  {
>    int ret = 0;
> -  CLEANUP_FCLOSE FILE *fp = NULL;
>    CLEANUP_UNLINK_FREE char *tmpfile = NULL;
> 
> -  ret = guestfs_int_lazy_make_tmpdir (g);
> -  if (ret < 0)
> -    return NULL;
> -
> -  tmpfile = safe_asprintf (g, "%s/filesystem_walk%d",
g->tmpdir, ++g->unique);
> +  tmpfile = make_temp_file (g, "filesystem_walk");
Note you are not checking the return value of make_temp_file, which can
still fail (when guestfs_int_lazy_make_tmpdir fails, for example).
So you still need to check tmpfile, otherwise tmpfile may be used as
null pointer.

Thanks,
-- 
Pino Toscano

On Saturday, 17 September 2016 18:18:52 CEST Matteo Cafasso
wrote:
> This series is ready for review but requires the previous one to be merged
first:
> https://www.redhat.com/archives/libguestfs/2016-September/msg00101.html
> 
> The find_block API allows the User to search all the filesystem entries
> referring to a given data block and returns a tsk_dirent structure
> for each of them.
> 
> Use case examples:
> 
>  - Check whether a block containing a deleted file has been re-used to
store a new one.
>  - Map a certain area of a disk with the contained files.
> 
> Matteo Cafasso (4):
>   New API: internal_find_block
>   New API: find_block
>   find_block: added API tests
>   TSK: small refactoring
Minus the small note in patch #1, patches #1-#3 LGTM.

Thanks,
-- 
Pino Toscano