Libguestfs - Dec 2013 - Re: Regression testing inspection code

[Continuing a discussion we had on IRC last week ...]

How to create meaningful regression tests for the inspection code?
We'd like developers to be able to test that changes to the inspection
code don't break inspection of existing guests.  We might not be able
to run these tests routinely, and we might have to accept that they
have to download hundreds of megabytes of test data.


Idea #1: Run virt-inspector on cloud images
----------------------------------------------------------------------

eg:

virt-inspector -a
http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2

Unfortunately this doesn't work for a couple of reasons.  Firstly the
Curl driver in qemu is broken (RHBZ#971790).  Secondly even when I
implemented a workaround, the command above doesn't work reliably when
connecting to a slow mirror, because of SCSI timeouts inside the
appliance.


Idea #2: Make "reduced" test images available
----------------------------------------------------------------------

Delete bits of real test images so they are small enough to
distribute.  Inspection doesn't look at the vast majority of files in
a test image.  The hard bit is working out which bits inspection
*does* need to look at, either now or in the future.

I wrote a small program to test this.  Unfortunately it doesn't reduce
images that much.  We'd still be talking about ~50-100MB to host +
distribute per test case, and we'd ideally want >20 test cases so
that's still multiple gigabytes of test data.

I've attached the program to this email in case anyone wants to look
at this further.


Any other ideas ...?

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines.  Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://people.redhat.com/~rjones/virt-top
-------------- next part --------------
#!/usr/bin/env ocaml

#load "unix.cma";;
#directory "+guestfs";;
#load "mlguestfs.cma";;

open Printf
module G = Guestfs

let (//) = Filename.concat
let quote = Filename.quote
let (+^) = Int64.add
let (-^) = Int64.sub
let ( *^ ) = Int64.mul
let (/^) = Int64.div

type ('a,'b) alternate = Either of 'a | Or of 'b
type visit_r = OK | Prune

let rec main ()   if Array.length Sys.argv < 2 then (
    eprintf "usage: %s disk [output.disk]\n" Sys.executable_name;
    exit 1
  );
  let disk = Sys.argv.(1) in

  (* Make an overlay over the input disk. *)
  let overlay = Filename.temp_file "ovl" ".qcow2" in
  let cmd     sprintf "qemu-img create -f qcow2 -b %s %s" (quote disk)
(quote overlay) in
  prerr_endline cmd;
  let i = Sys.command cmd in
  if i <> 0 then exit i;

  let g = new G.guestfs () in
  g#add_drive_opts ~format:"qcow2" overlay;
  g#launch ();

  (* Estimate the size of the original disk. *)
  let orig_size = estimate_size g in
  printf "original size [uncompressed, estimate]: %s\n%!"
    (human_size orig_size);

  (* Mount up the filesystems. *)
  inspect_and_mount g;

  (* Get "inspection signature" before removing anything. *)
  let orig_sig = inspection_signature g in

  (* Start deleting things. *)
  remove g "/home";
  remove g "/opt";
  remove g "/usr/etc";
  remove g "/usr/games";
  remove g "/usr/include";
  remove g "/usr/lib";
  remove g "/usr/lib64";
  remove g "/usr/libexec";
  remove g "/usr/local";
  remove g "/usr/share" ~except:[
    "/usr/share/cirros";
    "/usr/share/icons";
    "/usr/share/pixmaps";
  ];
  remove g "/usr/src";
  remove g "/var" ~except:[
    "/var/lib"                          (* /var/lib for package dbs *)
  ];

  g#umount_all ();

  (* Get "inspection signature" after. *)
  inspect_and_mount g;
  let final_sig = inspection_signature g in
  g#umount_all ();

  if orig_sig <> final_sig then (
    eprintf "oops, that changed the results of inspection\n";
    exit 1
  );

  (* Estimate the size of the final disk. *)
  let final_size = estimate_size g in
  printf "final size [uncompressed, estimate]: %s\n%!" (human_size
final_size);

  (* If there's an output disk, then run virt-sparsify and xz to
   * generate the final image.
   *)
  if Array.length Sys.argv >= 3 then (
    let output = Sys.argv.(2) in

    let cmd       sprintf "virt-sparsify --quiet --format qcow2 %s
--convert raw %s"
        (quote overlay) (quote output) in
    prerr_endline cmd;
    let i = Sys.command cmd in
    if i <> 0 then exit i;

    let cmd = sprintf "du -h %s" (quote output) in
    prerr_endline cmd;
    let i = Sys.command cmd in
    if i <> 0 then exit i;

    let cmd = sprintf "rm -f %s.xz; xz --best --block-size=%Ld %s"
      (quote output) (16L *^ 1024L *^ 1024L) (quote output) in
    prerr_endline cmd;
    let i = Sys.command cmd in
    if i <> 0 then exit i;

    let cmd = sprintf "ls -lh %s.xz" (quote output) in
    prerr_endline cmd;
    let i = Sys.command cmd in
    if i <> 0 then exit i
  );

  Unix.unlink overlay

(* Estimate the size of a disk, uncompressed bytes, after using
 * virt-sparsify.
 *)
and estimate_size g   let size = ref 0L in

  (* Virt-sparsify is usually able to recover any unused space in VGs,
   * so we don't have to examine that.  Just look at filesystems.
   *)
  let fses = g#list_filesystems () in
  List.iter (
    fun (dev, typ) ->
      (* Assume swap contains random data and virt-sparsify would
       * be able to replace it with a blank swap partition.
       *)
      if typ <> "swap" then (
        let devsize = g#blockdev_getsize64 dev in
        let mounted = try g#mount_ro dev "/"; true with G.Error _
-> false in

        (* Assume virt-sparsify wouldn't be able to touch an unmountable
dev. *)
        if not mounted then
          size := !size +^ devsize
        else (
          (* For a mountable filesystem, assume virt-sparsify would be
           * able to recover all unused space.
           *)
          let statvfs = g#statvfs "/" in
          size := !size +^ (devsize -^ (statvfs.G.bsize *^ statvfs.G.bfree));

          g#umount_all ()
        )
      )
  ) fses;

  !size

and human_size i   let sign, i = if i < 0L then "-", Int64.neg i
else "", i in

  if i < 1024L then
    sprintf "%s%Ld" sign i
  else (
    let f = Int64.to_float i /. 1024. in
    let i = i /^ 1024L in
    if i < 1024L then
      sprintf "%s%.1fK" sign f
    else (
      let f = Int64.to_float i /. 1024. in
      let i = i /^ 1024L in
      if i < 1024L then
        sprintf "%s%.1fM" sign f
      else (
        let f = Int64.to_float i /. 1024. in
        (*let i = i /^ 1024L in*)
        sprintf "%s%.1fG" sign f
      )
    )
  )

(* This is like rm -rf except:
 * - It doesn't delete directories or symlinks.
 * - The optional ?except parameter can be used to "save" a prefix
from
 *   being deleted.
 *)
and remove g ?(except = []) path   if g#exists path then (
    if g#is_dir ~followsymlinks:false path then (
      let rec f pathname _ _ { G.mode = mode }         if is_except pathname
then
          Prune                         (* prune ?except subtrees *)
        else if is_dir mode then OK     (* don't delete directories *)
        else if is_lnk mode then OK     (* don't delete symlinks *)
        else (
          g#rm pathname;                (* otherwise delete it *)
          OK
        )
      and is_except pathname         List.exists (
          fun prefix -> starts_with pathname prefix
        ) except
      and starts_with pathname prefix         let pathnamelen = String.length
pathname in
        let prefixlen = String.length prefix in
        prefixlen <= pathnamelen &&
          String.sub pathname 0 prefixlen = prefix
      in

      visit g ~f path
    ) else
      g#rm path
  )

(* See cat/ls.c: visit() *)
and visit g ~f ?(depth = 0) dir   (* Call the callback function on the top level
directory. *)
  let r     if depth = 0 then (
      let stat = g#lstat dir in
      f dir dir None stat
    )
    else OK in

  match r with
  | Prune -> ()
  | OK ->
    let names = g#ls dir in
    let stats = g#lstatlist dir names in

    assert (Array.length names = Array.length stats);

    for i = 0 to Array.length names - 1 do
      let path         if dir = "/" then "/" ^ names.(i)
        else dir ^ "/" ^ names.(i) in
      let r = f path dir (Some names.(i)) stats.(i) in

      if r = OK && is_dir stats.(i).G.mode then
        visit g ~f ~depth:(depth+1) path
    done

and is_dir mode = Int64.logand mode 0o170_000L = 0o040_000L

and is_lnk mode = Int64.logand mode 0o170_000L = 0o120_000L

and inspect_and_mount g   let roots = g#inspect_os () in
  if Array.length roots <> 1 then
    failwith "inspect_vm: no operating systems or multi-boot guest";

  let root = roots.(0) in

  let mps = g#inspect_get_mountpoints root in
  let cmp (a,_) (b,_) = compare (String.length a) (String.length b) in
  let mps = List.sort cmp mps in
  List.iter (
    fun (mp, dev) ->
      try g#mount dev mp with G.Error msg -> eprintf "%s
(ignored)\n%!" msg
  ) mps

(* This lets us check that no parts of inspection change before
 * and after removing bits from the guest.
 *)
and inspection_signature g   let root = (g#inspect_get_roots ()).(0) in
  (g#inspect_get_arch root,
   g#inspect_get_distro root,
   g#inspect_get_drive_mappings root,
   g#inspect_get_filesystems root,
   g#inspect_get_format root,
   g#inspect_get_hostname root,
   g#inspect_get_icon root,
   g#inspect_get_major_version root,
   g#inspect_get_minor_version root,
   g#inspect_get_mountpoints root,
   g#inspect_get_package_format root,
   g#inspect_get_package_management root,
   g#inspect_get_product_name root,
   g#inspect_get_product_variant root,
   g#inspect_get_type root,
   (try Either (g#inspect_get_windows_current_control_set root)
    with G.Error msg -> Or msg),
   (try Either (g#inspect_get_windows_systemroot root)
    with G.Error msg -> Or msg),
   g#inspect_is_live root,
   g#inspect_is_multipart root,
   g#inspect_is_netinst root,
   g#inspect_list_applications2 root)

let () = main ()

On Mon, 2013-12-02 at 13:23 +0000, Richard W.M. Jones
wrote:
> [Continuing a discussion we had on IRC last week ...]
> 
> How to create meaningful regression tests for the inspection code?
> We'd like developers to be able to test that changes to the inspection
> code don't break inspection of existing guests.  We might not be able
> to run these tests routinely, and we might have to accept that they
> have to download hundreds of megabytes of test data.
> 
> 
> Idea #1: Run virt-inspector on cloud images
> ----------------------------------------------------------------------
> 
> eg:
> 
> virt-inspector -a
http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2
> 
> Unfortunately this doesn't work for a couple of reasons.  Firstly the
> Curl driver in qemu is broken (RHBZ#971790).  Secondly even when I
> implemented a workaround, the command above doesn't work reliably when
> connecting to a slow mirror, because of SCSI timeouts inside the
> appliance.
> 
> 
> Idea #2: Make "reduced" test images available
> ----------------------------------------------------------------------
> 
> Delete bits of real test images so they are small enough to
> distribute.  Inspection doesn't look at the vast majority of files in
> a test image.  The hard bit is working out which bits inspection
> *does* need to look at, either now or in the future.
> 
> I wrote a small program to test this.  Unfortunately it doesn't reduce
> images that much.  We'd still be talking about ~50-100MB to host +
> distribute per test case, and we'd ideally want >20 test cases so
> that's still multiple gigabytes of test data.
> 
> I've attached the program to this email in case anyone wants to look
> at this further.
> 
> 
> Any other ideas ...?
guestconv builds several RHEL and Fedora based images for automated
testing. The build process is separate from the test process, and tests
are skipped if their image hasn't been built. It's very flexible, and
works for guests we can build automatically.

Matt

On Mon, Dec 02, 2013 at 01:33:53PM +0000, Matthew Booth
wrote:
> On Mon, 2013-12-02 at 13:23 +0000, Richard W.M. Jones wrote:
> > [Continuing a discussion we had on IRC last week ...]
> > 
> > How to create meaningful regression tests for the inspection code?
> > We'd like developers to be able to test that changes to the
inspection
> > code don't break inspection of existing guests.  We might not be
able
> > to run these tests routinely, and we might have to accept that they
> > have to download hundreds of megabytes of test data.
> > 
> > 
> > Idea #1: Run virt-inspector on cloud images
> > ----------------------------------------------------------------------
> > 
> > eg:
> > 
> > virt-inspector -a
http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2
> > 
> > Unfortunately this doesn't work for a couple of reasons.  Firstly
the
> > Curl driver in qemu is broken (RHBZ#971790).  Secondly even when I
> > implemented a workaround, the command above doesn't work reliably
when
> > connecting to a slow mirror, because of SCSI timeouts inside the
> > appliance.
> > 
> > 
> > Idea #2: Make "reduced" test images available
> > ----------------------------------------------------------------------
> > 
> > Delete bits of real test images so they are small enough to
> > distribute.  Inspection doesn't look at the vast majority of files
in
> > a test image.  The hard bit is working out which bits inspection
> > *does* need to look at, either now or in the future.
> > 
> > I wrote a small program to test this.  Unfortunately it doesn't
reduce
> > images that much.  We'd still be talking about ~50-100MB to host +
> > distribute per test case, and we'd ideally want >20 test cases
so
> > that's still multiple gigabytes of test data.
> > 
> > I've attached the program to this email in case anyone wants to
look
> > at this further.
> > 
> > 
> > Any other ideas ...?
> 
> guestconv builds several RHEL and Fedora based images for automated
> testing. The build process is separate from the test process, and tests
> are skipped if their image hasn't been built. It's very flexible,
and
> works for guests we can build automatically.
Can you link to the code?  Does it use Oz?

Note the really interesting regression test cases (for inspection, not
for v2v) are obscure Linux distros, *BSD, etc.  These are the ones
which are likely to break without anyone noticing.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine.  Supports Linux and Windows.
http://people.redhat.com/~rjones/virt-df/