[Continuing a discussion we had on IRC last week ...]
How to create meaningful regression tests for the inspection code?
We'd like developers to be able to test that changes to the inspection
code don't break inspection of existing guests. We might not be able
to run these tests routinely, and we might have to accept that they
have to download hundreds of megabytes of test data.
Idea #1: Run virt-inspector on cloud images
----------------------------------------------------------------------
eg:
virt-inspector -a
http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2
Unfortunately this doesn't work for a couple of reasons. Firstly the
Curl driver in qemu is broken (RHBZ#971790). Secondly even when I
implemented a workaround, the command above doesn't work reliably when
connecting to a slow mirror, because of SCSI timeouts inside the
appliance.
Idea #2: Make "reduced" test images available
----------------------------------------------------------------------
Delete bits of real test images so they are small enough to
distribute. Inspection doesn't look at the vast majority of files in
a test image. The hard bit is working out which bits inspection
*does* need to look at, either now or in the future.
I wrote a small program to test this. Unfortunately it doesn't reduce
images that much. We'd still be talking about ~50-100MB to host +
distribute per test case, and we'd ideally want >20 test cases so
that's still multiple gigabytes of test data.
I've attached the program to this email in case anyone wants to look
at this further.
Any other ideas ...?
Rich.
--
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
virt-top is 'top' for virtual machines. Tiny program with many
powerful monitoring features, net stats, disk stats, logging, etc.
http://people.redhat.com/~rjones/virt-top
-------------- next part --------------
#!/usr/bin/env ocaml
#load "unix.cma";;
#directory "+guestfs";;
#load "mlguestfs.cma";;
open Printf
module G = Guestfs
let (//) = Filename.concat
let quote = Filename.quote
let (+^) = Int64.add
let (-^) = Int64.sub
let ( *^ ) = Int64.mul
let (/^) = Int64.div
type ('a,'b) alternate = Either of 'a | Or of 'b
type visit_r = OK | Prune
let rec main () if Array.length Sys.argv < 2 then (
eprintf "usage: %s disk [output.disk]\n" Sys.executable_name;
exit 1
);
let disk = Sys.argv.(1) in
(* Make an overlay over the input disk. *)
let overlay = Filename.temp_file "ovl" ".qcow2" in
let cmd sprintf "qemu-img create -f qcow2 -b %s %s" (quote disk)
(quote overlay) in
prerr_endline cmd;
let i = Sys.command cmd in
if i <> 0 then exit i;
let g = new G.guestfs () in
g#add_drive_opts ~format:"qcow2" overlay;
g#launch ();
(* Estimate the size of the original disk. *)
let orig_size = estimate_size g in
printf "original size [uncompressed, estimate]: %s\n%!"
(human_size orig_size);
(* Mount up the filesystems. *)
inspect_and_mount g;
(* Get "inspection signature" before removing anything. *)
let orig_sig = inspection_signature g in
(* Start deleting things. *)
remove g "/home";
remove g "/opt";
remove g "/usr/etc";
remove g "/usr/games";
remove g "/usr/include";
remove g "/usr/lib";
remove g "/usr/lib64";
remove g "/usr/libexec";
remove g "/usr/local";
remove g "/usr/share" ~except:[
"/usr/share/cirros";
"/usr/share/icons";
"/usr/share/pixmaps";
];
remove g "/usr/src";
remove g "/var" ~except:[
"/var/lib" (* /var/lib for package dbs *)
];
g#umount_all ();
(* Get "inspection signature" after. *)
inspect_and_mount g;
let final_sig = inspection_signature g in
g#umount_all ();
if orig_sig <> final_sig then (
eprintf "oops, that changed the results of inspection\n";
exit 1
);
(* Estimate the size of the final disk. *)
let final_size = estimate_size g in
printf "final size [uncompressed, estimate]: %s\n%!" (human_size
final_size);
(* If there's an output disk, then run virt-sparsify and xz to
* generate the final image.
*)
if Array.length Sys.argv >= 3 then (
let output = Sys.argv.(2) in
let cmd sprintf "virt-sparsify --quiet --format qcow2 %s
--convert raw %s"
(quote overlay) (quote output) in
prerr_endline cmd;
let i = Sys.command cmd in
if i <> 0 then exit i;
let cmd = sprintf "du -h %s" (quote output) in
prerr_endline cmd;
let i = Sys.command cmd in
if i <> 0 then exit i;
let cmd = sprintf "rm -f %s.xz; xz --best --block-size=%Ld %s"
(quote output) (16L *^ 1024L *^ 1024L) (quote output) in
prerr_endline cmd;
let i = Sys.command cmd in
if i <> 0 then exit i;
let cmd = sprintf "ls -lh %s.xz" (quote output) in
prerr_endline cmd;
let i = Sys.command cmd in
if i <> 0 then exit i
);
Unix.unlink overlay
(* Estimate the size of a disk, uncompressed bytes, after using
* virt-sparsify.
*)
and estimate_size g let size = ref 0L in
(* Virt-sparsify is usually able to recover any unused space in VGs,
* so we don't have to examine that. Just look at filesystems.
*)
let fses = g#list_filesystems () in
List.iter (
fun (dev, typ) ->
(* Assume swap contains random data and virt-sparsify would
* be able to replace it with a blank swap partition.
*)
if typ <> "swap" then (
let devsize = g#blockdev_getsize64 dev in
let mounted = try g#mount_ro dev "/"; true with G.Error _
-> false in
(* Assume virt-sparsify wouldn't be able to touch an unmountable
dev. *)
if not mounted then
size := !size +^ devsize
else (
(* For a mountable filesystem, assume virt-sparsify would be
* able to recover all unused space.
*)
let statvfs = g#statvfs "/" in
size := !size +^ (devsize -^ (statvfs.G.bsize *^ statvfs.G.bfree));
g#umount_all ()
)
)
) fses;
!size
and human_size i let sign, i = if i < 0L then "-", Int64.neg i
else "", i in
if i < 1024L then
sprintf "%s%Ld" sign i
else (
let f = Int64.to_float i /. 1024. in
let i = i /^ 1024L in
if i < 1024L then
sprintf "%s%.1fK" sign f
else (
let f = Int64.to_float i /. 1024. in
let i = i /^ 1024L in
if i < 1024L then
sprintf "%s%.1fM" sign f
else (
let f = Int64.to_float i /. 1024. in
(*let i = i /^ 1024L in*)
sprintf "%s%.1fG" sign f
)
)
)
(* This is like rm -rf except:
* - It doesn't delete directories or symlinks.
* - The optional ?except parameter can be used to "save" a prefix
from
* being deleted.
*)
and remove g ?(except = []) path if g#exists path then (
if g#is_dir ~followsymlinks:false path then (
let rec f pathname _ _ { G.mode = mode } if is_except pathname
then
Prune (* prune ?except subtrees *)
else if is_dir mode then OK (* don't delete directories *)
else if is_lnk mode then OK (* don't delete symlinks *)
else (
g#rm pathname; (* otherwise delete it *)
OK
)
and is_except pathname List.exists (
fun prefix -> starts_with pathname prefix
) except
and starts_with pathname prefix let pathnamelen = String.length
pathname in
let prefixlen = String.length prefix in
prefixlen <= pathnamelen &&
String.sub pathname 0 prefixlen = prefix
in
visit g ~f path
) else
g#rm path
)
(* See cat/ls.c: visit() *)
and visit g ~f ?(depth = 0) dir (* Call the callback function on the top level
directory. *)
let r if depth = 0 then (
let stat = g#lstat dir in
f dir dir None stat
)
else OK in
match r with
| Prune -> ()
| OK ->
let names = g#ls dir in
let stats = g#lstatlist dir names in
assert (Array.length names = Array.length stats);
for i = 0 to Array.length names - 1 do
let path if dir = "/" then "/" ^ names.(i)
else dir ^ "/" ^ names.(i) in
let r = f path dir (Some names.(i)) stats.(i) in
if r = OK && is_dir stats.(i).G.mode then
visit g ~f ~depth:(depth+1) path
done
and is_dir mode = Int64.logand mode 0o170_000L = 0o040_000L
and is_lnk mode = Int64.logand mode 0o170_000L = 0o120_000L
and inspect_and_mount g let roots = g#inspect_os () in
if Array.length roots <> 1 then
failwith "inspect_vm: no operating systems or multi-boot guest";
let root = roots.(0) in
let mps = g#inspect_get_mountpoints root in
let cmp (a,_) (b,_) = compare (String.length a) (String.length b) in
let mps = List.sort cmp mps in
List.iter (
fun (mp, dev) ->
try g#mount dev mp with G.Error msg -> eprintf "%s
(ignored)\n%!" msg
) mps
(* This lets us check that no parts of inspection change before
* and after removing bits from the guest.
*)
and inspection_signature g let root = (g#inspect_get_roots ()).(0) in
(g#inspect_get_arch root,
g#inspect_get_distro root,
g#inspect_get_drive_mappings root,
g#inspect_get_filesystems root,
g#inspect_get_format root,
g#inspect_get_hostname root,
g#inspect_get_icon root,
g#inspect_get_major_version root,
g#inspect_get_minor_version root,
g#inspect_get_mountpoints root,
g#inspect_get_package_format root,
g#inspect_get_package_management root,
g#inspect_get_product_name root,
g#inspect_get_product_variant root,
g#inspect_get_type root,
(try Either (g#inspect_get_windows_current_control_set root)
with G.Error msg -> Or msg),
(try Either (g#inspect_get_windows_systemroot root)
with G.Error msg -> Or msg),
g#inspect_is_live root,
g#inspect_is_multipart root,
g#inspect_is_netinst root,
g#inspect_list_applications2 root)
let () = main ()
On Mon, 2013-12-02 at 13:23 +0000, Richard W.M. Jones wrote:> [Continuing a discussion we had on IRC last week ...] > > How to create meaningful regression tests for the inspection code? > We'd like developers to be able to test that changes to the inspection > code don't break inspection of existing guests. We might not be able > to run these tests routinely, and we might have to accept that they > have to download hundreds of megabytes of test data. > > > Idea #1: Run virt-inspector on cloud images > ---------------------------------------------------------------------- > > eg: > > virt-inspector -a http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2 > > Unfortunately this doesn't work for a couple of reasons. Firstly the > Curl driver in qemu is broken (RHBZ#971790). Secondly even when I > implemented a workaround, the command above doesn't work reliably when > connecting to a slow mirror, because of SCSI timeouts inside the > appliance. > > > Idea #2: Make "reduced" test images available > ---------------------------------------------------------------------- > > Delete bits of real test images so they are small enough to > distribute. Inspection doesn't look at the vast majority of files in > a test image. The hard bit is working out which bits inspection > *does* need to look at, either now or in the future. > > I wrote a small program to test this. Unfortunately it doesn't reduce > images that much. We'd still be talking about ~50-100MB to host + > distribute per test case, and we'd ideally want >20 test cases so > that's still multiple gigabytes of test data. > > I've attached the program to this email in case anyone wants to look > at this further. > > > Any other ideas ...?guestconv builds several RHEL and Fedora based images for automated testing. The build process is separate from the test process, and tests are skipped if their image hasn't been built. It's very flexible, and works for guests we can build automatically. Matt
Richard W.M. Jones
2013-Dec-02 13:45 UTC
Re: [Libguestfs] Regression testing inspection code
On Mon, Dec 02, 2013 at 01:33:53PM +0000, Matthew Booth wrote:> On Mon, 2013-12-02 at 13:23 +0000, Richard W.M. Jones wrote: > > [Continuing a discussion we had on IRC last week ...] > > > > How to create meaningful regression tests for the inspection code? > > We'd like developers to be able to test that changes to the inspection > > code don't break inspection of existing guests. We might not be able > > to run these tests routinely, and we might have to accept that they > > have to download hundreds of megabytes of test data. > > > > > > Idea #1: Run virt-inspector on cloud images > > ---------------------------------------------------------------------- > > > > eg: > > > > virt-inspector -a http://download.fedoraproject.org/pub/fedora/linux/releases/19/Images/x86_64/Fedora-x86_64-19-20130627-sda.qcow2 > > > > Unfortunately this doesn't work for a couple of reasons. Firstly the > > Curl driver in qemu is broken (RHBZ#971790). Secondly even when I > > implemented a workaround, the command above doesn't work reliably when > > connecting to a slow mirror, because of SCSI timeouts inside the > > appliance. > > > > > > Idea #2: Make "reduced" test images available > > ---------------------------------------------------------------------- > > > > Delete bits of real test images so they are small enough to > > distribute. Inspection doesn't look at the vast majority of files in > > a test image. The hard bit is working out which bits inspection > > *does* need to look at, either now or in the future. > > > > I wrote a small program to test this. Unfortunately it doesn't reduce > > images that much. We'd still be talking about ~50-100MB to host + > > distribute per test case, and we'd ideally want >20 test cases so > > that's still multiple gigabytes of test data. > > > > I've attached the program to this email in case anyone wants to look > > at this further. > > > > > > Any other ideas ...? > > guestconv builds several RHEL and Fedora based images for automated > testing. The build process is separate from the test process, and tests > are skipped if their image hasn't been built. It's very flexible, and > works for guests we can build automatically.Can you link to the code? Does it use Oz? Note the really interesting regression test cases (for inspection, not for v2v) are obscure Linux distros, *BSD, etc. These are the ones which are likely to break without anyone noticing. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://people.redhat.com/~rjones/virt-df/