Libguestfs - Sep 2015 - [PATCH 4/4] builder: support Simple Streams v1.0 as index metadata

Hi,

this series adds a basic support for Simple Streams v1.0 metadata
files. This makes it possible to create a repository .conf files with

 [cirros]
 uri=http://download.cirros-cloud.net
 format=simplestreams

to read the latest version of each CirrOS image.

Thanks,


Pino Toscano (4):
  builder: add non-int revisions
  builder: add simple libyajl binding
  build: expose HAVE_YAJL to automake
  builder: support Simple Streams v1.0 as index metadata

 .gitignore                                         |   3 +
 builder/Makefile.am                                |  51 ++-
 builder/builder.ml                                 |  11 +-
 builder/cache.ml                                   |   2 +-
 builder/cache.mli                                  |   6 +-
 builder/downloader.mli                             |   2 +-
 builder/index.ml                                   |   4 +-
 builder/index.mli                                  |   2 +-
 builder/index_parser.ml                            |   4 +-
 builder/simplestreams_parser.ml                    | 204 ++++++++++
 builder/simplestreams_parser.mli                   |  19 +
 builder/sources.ml                                 |   9 +
 builder/sources.mli                                |   1 +
 builder/test-simplestreams/streams/v1/index.json   |  18 +
 .../v1/net.cirros-cloud:released:download.json     | 429 +++++++++++++++++++++
 .../virt-builder/repos.d/cirros.conf.in            |   3 +
 builder/test-virt-builder-list-simplestreams.sh    | 108 ++++++
 builder/utils.ml                                   |   7 +
 builder/virt-builder.pod                           |   7 +
 builder/yajl-c.c                                   | 143 +++++++
 builder/yajl.ml                                    |  30 ++
 builder/yajl.mli                                   |  33 ++
 builder/yajl_tests.ml                              | 139 +++++++
 configure.ac                                       |   2 +
 po/POTFILES                                        |   1 +
 po/POTFILES-ml                                     |   3 +
 26 files changed, 1226 insertions(+), 15 deletions(-)
 create mode 100644 builder/simplestreams_parser.ml
 create mode 100644 builder/simplestreams_parser.mli
 create mode 100644 builder/test-simplestreams/streams/v1/index.json
 create mode 100644
builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json
 create mode 100644
builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in
 create mode 100755 builder/test-virt-builder-list-simplestreams.sh
 create mode 100644 builder/yajl-c.c
 create mode 100644 builder/yajl.ml
 create mode 100644 builder/yajl.mli
 create mode 100644 builder/yajl_tests.ml

-- 
2.1.0

Add support for non-integer revisions of entries, comparing them as
integer when possible.
---
 builder/builder.ml      | 9 ++++++++-
 builder/cache.ml        | 2 +-
 builder/cache.mli       | 6 +++---
 builder/downloader.mli  | 2 +-
 builder/index.ml        | 4 ++--
 builder/index.mli       | 2 +-
 builder/index_parser.ml | 4 ++--
 builder/utils.ml        | 7 +++++++
 8 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index d59380b..dcfd437 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -34,6 +34,13 @@ open Printf
 let () = Random.self_init ()
 
 let remove_duplicates index +  let compare_revisions rev1 rev2 +    match rev1,
rev2 with
+    | Rev_int n1, Rev_int n2 -> compare n1 n2
+    | Rev_string s1, Rev_int n2 -> compare s1 (string_of_int n2)
+    | Rev_int n1, Rev_string s2 -> compare (string_of_int n1) s2
+    | Rev_string s1, Rev_string s2 -> compare s1 s2
+  in
   (* Fill an hash with the higher revision of the available
    * (name, arch) tuples, so it possible to ignore duplicates,
    * and versions with a lower revision.
@@ -44,7 +51,7 @@ let remove_duplicates index        let id = name, arch in
       try
         let rev = Hashtbl.find nseen id in
-        if revision > rev then
+        if compare_revisions rev revision > 0 then
           Hashtbl.replace nseen id revision
       with Not_found ->
         Hashtbl.add nseen id revision
diff --git a/builder/cache.ml b/builder/cache.ml
index e73bcfd..8b63a86 100644
--- a/builder/cache.ml
+++ b/builder/cache.ml
@@ -40,7 +40,7 @@ let create ~directory    }
 
 let cache_of_name t name arch revision -  t.directory // sprintf
"%s.%s.%d" name arch revision
+  t.directory // sprintf "%s.%s.%s" name arch (string_of_revision
revision)
 
 let is_cached t name arch revision    let filename = cache_of_name t name arch
revision in
diff --git a/builder/cache.mli b/builder/cache.mli
index 7edc670..1ab8ede 100644
--- a/builder/cache.mli
+++ b/builder/cache.mli
@@ -27,16 +27,16 @@ type t
 val create : directory:string -> t
 (** Create the abstract type. *)
 
-val cache_of_name : t -> string -> string -> int -> string
+val cache_of_name : t -> string -> string -> Utils.revision ->
string
 (** [cache_of_name t name arch revision] return the filename
     of the cached file.  (Note: It doesn't check if the filename
     exists, this is just a simple string transformation). *)
 
-val is_cached : t -> string -> string -> int -> bool
+val is_cached : t -> string -> string -> Utils.revision -> bool
 (** [is_cached t name arch revision] return whether the file with
     specified name, architecture and revision is cached. *)
 
-val print_item_status : t -> header:bool -> (string * string * int) list
-> unit
+val print_item_status : t -> header:bool -> (string * string *
Utils.revision) list -> unit
 (** [print_item_status t header items] print the status in the cache
     of the specified items (which are tuples of name, architecture,
     and revision).
diff --git a/builder/downloader.mli b/builder/downloader.mli
index 5e3cdaa..11ec498 100644
--- a/builder/downloader.mli
+++ b/builder/downloader.mli
@@ -35,7 +35,7 @@ type proxy_mode  val create : curl:string -> cache:Cache.t
option -> t
 (** Create the abstract type. *)
 
-val download : t -> ?template:(string*string*int) -> ?progress_bar:bool
-> ?proxy:proxy_mode -> uri -> (filename * bool)
+val download : t -> ?template:(string*string*Utils.revision) ->
?progress_bar:bool -> ?proxy:proxy_mode -> uri -> (filename * bool)
 (** Download the URI, returning the downloaded filename and a
     temporary file flag.  The temporary file flag is [true] iff
     the downloaded file is temporary and should be deleted by the
diff --git a/builder/index.ml b/builder/index.ml
index 3e8cb85..c59d6dd 100644
--- a/builder/index.ml
+++ b/builder/index.ml
@@ -32,7 +32,7 @@ and entry = {
   arch : string;
   signature_uri : string option;        (* deprecated, will be removed in 1.26
*)
   checksums : Checksums.csum_t list option;
-  revision : int;
+  revision : Utils.revision;
   format : string option;
   size : int64;
   compressed_size : int64 option;
@@ -86,7 +86,7 @@ let print_entry chan (name, { printable_name = printable_name;
           (Checksums.string_of_csum_t c) (Checksums.string_of_csum c)
     ) checksums
   );
-  fp "revision=%d\n" revision;
+  fp "revision=%s\n" (string_of_revision revision);
   (match format with
   | None -> ()
   | Some format -> fp "format=%s\n" format
diff --git a/builder/index.mli b/builder/index.mli
index 10ed15a..fadcad9 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -24,7 +24,7 @@ and entry = {
   arch : string;
   signature_uri : string option;        (* deprecated, will be removed in 1.26
*)
   checksums : Checksums.csum_t list option;
-  revision : int;
+  revision : Utils.revision;
   format : string option;
   size : int64;
   compressed_size : int64 option;
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 845d0e9..2c78fd9 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -112,9 +112,9 @@ let get_index ~downloader ~sigchecker
               try Some (List.assoc ("checksum", None) fields)
               with Not_found -> None in
           let revision -            try int_of_string (List.assoc
("revision", None) fields)
+            try Rev_int (int_of_string (List.assoc ("revision", None)
fields))
             with
-            | Not_found -> 1
+            | Not_found -> Rev_int 1
             | Failure "int_of_string" ->
               eprintf (f_"%s: cannot parse 'revision' field for
'%s'\n") prog n;
               corrupt_file () in
diff --git a/builder/utils.ml b/builder/utils.ml
index a6628eb..986bf68 100644
--- a/builder/utils.ml
+++ b/builder/utils.ml
@@ -26,5 +26,12 @@ type gpgkey_type    | No_Key
   | Fingerprint of string
   | KeyFile of string
+and revision +  | Rev_int of int
+  | Rev_string of string
 
 let quote = Filename.quote
+
+let string_of_revision = function
+  | Rev_int n -> string_of_int n
+  | Rev_string s -> s
-- 
2.1.0

Only yajl_val and yajl_tree_parse are exposed for now.
---
 .gitignore            |   2 +
 builder/Makefile.am   |  45 ++++++++++++++--
 builder/yajl-c.c      | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++
 builder/yajl.ml       |  30 +++++++++++
 builder/yajl.mli      |  33 ++++++++++++
 builder/yajl_tests.ml | 139 ++++++++++++++++++++++++++++++++++++++++++++++++
 po/POTFILES           |   1 +
 po/POTFILES-ml        |   2 +
 8 files changed, 391 insertions(+), 4 deletions(-)
 create mode 100644 builder/yajl-c.c
 create mode 100644 builder/yajl.ml
 create mode 100644 builder/yajl.mli
 create mode 100644 builder/yajl_tests.ml

diff --git a/.gitignore b/.gitignore
index e502018..db8d0a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,6 +60,7 @@ Makefile.in
 /builder/index-parse.h
 /builder/index-scan.c
 /builder/libguestfs.conf
+/builder/oUnit-*
 /builder/*.qcow2
 /builder/stamp-virt-builder.pod
 /builder/stamp-virt-index-validate.pod
@@ -70,6 +71,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
 /cat/virt-cat.1
diff --git a/builder/Makefile.am b/builder/Makefile.am
index f48efb0..366b8db 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -32,7 +32,8 @@ EXTRA_DIST = \
 	test-virt-index-validate-good-2 \
 	test-virt-index-validate-good-3 \
 	virt-builder.pod \
-	virt-index-validate.pod
+	virt-index-validate.pod \
+	yajl_tests.ml
 
 CLEANFILES = *~ *.annot *.cmi *.cmo *.cmx *.cmxa *.o virt-builder
 
@@ -48,7 +49,8 @@ SOURCES_MLI = \
 	pxzcat.mli \
 	setlocale.mli \
 	sigchecker.mli \
-	sources.mli
+	sources.mli \
+	yajl.mli
 
 SOURCES_ML = \
 	utils.ml \
@@ -57,6 +59,7 @@ SOURCES_ML = \
 	checksums.ml \
 	index.ml \
 	ini_reader.ml \
+	yajl.ml \
 	paths.ml \
 	languages.ml \
 	cache.ml \
@@ -81,7 +84,8 @@ SOURCES_C = \
 	index-parse.c \
 	index-parser-c.c \
 	pxzcat-c.c \
-	setlocale-c.c
+	setlocale-c.c \
+	yajl-c.c
 
 man_MANS  noinst_DATA @@ -106,7 +110,8 @@ virt_builder_CFLAGS = \
 	-Wno-unused-macros \
 	$(LIBLZMA_CFLAGS) \
 	$(LIBTINFO_CFLAGS) \
-	$(LIBXML2_CFLAGS)
+	$(LIBXML2_CFLAGS) \
+	$(YAJL_CFLAGS)
 
 BOBJECTS = \
 	$(top_builddir)/mllib/libdir.cmo \
@@ -145,9 +150,13 @@ OCAMLPACKAGES = \
 	-I $(top_builddir)/ocaml \
 	-I $(top_builddir)/mllib \
 	-I $(top_builddir)/customize
+OCAMLPACKAGES_TESTS  if HAVE_OCAML_PKG_GETTEXT
 OCAMLPACKAGES += -package gettext-stub
 endif
+if HAVE_OCAML_PKG_OUNIT
+OCAMLPACKAGES_TESTS += -package oUnit
+endif
 
 OCAMLCLIBS = \
 	-pthread -lpthread \
@@ -156,6 +165,7 @@ OCAMLCLIBS = \
 	$(LIBCRYPT_LIBS) \
 	$(LIBLZMA_LIBS) \
 	$(LIBXML2_LIBS) \
+	$(YAJL_LIBS) \
 	$(LIBINTL) \
 	-lgnu
 
@@ -232,13 +242,40 @@ fedora.qcow2.xz: fedora.qcow2
 	xz --best -c $< > $@-t
 	mv $@-t $@
 
+yajl_tests_SOURCES = yajl-c.c
+yajl_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+yajl_tests_BOBJECTS = \
+	yajl.cmo \
+	yajl_tests.cmo
+yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
+
+# Can't call the following as <test>_OBJECTS because automake gets
confused.
+if HAVE_OCAMLOPT
+yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
+yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+else
+yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
+yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+endif
+
+yajl_tests_DEPENDENCIES = $(yajl_tests_THEOBJECTS) $(top_srcdir)/ocaml-link.sh
+yajl_tests_LINK = \
+	$(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+	  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLPACKAGES_TESTS)
$(OCAMLLINKFLAGS) \
+	  $(yajl_tests_THEOBJECTS) -o $@
+
 TESTS = \
 	test-virt-builder-list.sh \
 	test-virt-index-validate.sh
+check_PROGRAMS  
 if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
+if HAVE_OCAML_PKG_OUNIT
+check_PROGRAMS += yajl_tests
+TESTS += yajl_tests
+endif
 
 check-valgrind:
 	$(MAKE) VG="$(top_builddir)/run @VG@" check
diff --git a/builder/yajl-c.c b/builder/yajl-c.c
new file mode 100644
index 0000000..6a96d59
--- /dev/null
+++ b/builder/yajl-c.c
@@ -0,0 +1,143 @@
+/* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA.
+ */
+
+#include <config.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#if HAVE_YAJL
+#include <yajl/yajl_tree.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+
+#define Val_none (Val_int (0))
+
+extern value virt_builder_yajl_is_available (value unit);
+extern value virt_builder_yajl_tree_parse (value stringv);
+
+#if HAVE_YAJL
+static value
+convert_yajl_value (yajl_val val, int level)
+{
+  CAMLparam0 ();
+  CAMLlocal4 (rv, lv, v, sv);
+
+  if (level > 20)
+    caml_invalid_argument ("too many levels of object/array
nesting");
+
+  if (YAJL_IS_OBJECT (val)) {
+    size_t len = YAJL_GET_OBJECT(val)->len;
+    size_t i;
+    rv = caml_alloc (1, 3);
+    lv = caml_alloc_tuple (len);
+    for (i = 0; i < len; ++i) {
+      v = caml_alloc_tuple (2);
+      sv = caml_copy_string (YAJL_GET_OBJECT(val)->keys[i]);
+      Store_field (v, 0, sv);
+      sv = convert_yajl_value (YAJL_GET_OBJECT(val)->values[i], level + 1);
+      Store_field (v, 1, sv);
+      Store_field (lv, i, v);
+    }
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_ARRAY (val)) {
+    size_t len = YAJL_GET_ARRAY(val)->len;
+    size_t i;
+    rv = caml_alloc (1, 4);
+    lv = caml_alloc_tuple (len);
+    for (i = 0; i < len; ++i) {
+      v = convert_yajl_value (YAJL_GET_ARRAY(val)->values[i], level + 1);
+      Store_field (lv, i, v);
+    }
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_STRING (val)) {
+    rv = caml_alloc (1, 0);
+    v = caml_copy_string (YAJL_GET_STRING(val));
+    Store_field (rv, 0, v);
+  } else if (YAJL_IS_DOUBLE (val)) {
+    rv = caml_alloc (1, 2);
+    lv = caml_alloc_tuple (1);
+    Store_double_field (lv, 0, YAJL_GET_DOUBLE(val));
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_INTEGER (val)) {
+    rv = caml_alloc (1, 1);
+    v = caml_copy_int64 (YAJL_GET_INTEGER(val));
+    Store_field (rv, 0, v);
+  } else if (YAJL_IS_TRUE (val)) {
+    rv = caml_alloc (1, 5);
+    Store_field (rv, 0, Val_true);
+  } else if (YAJL_IS_FALSE (val)) {
+    rv = caml_alloc (1, 5);
+    Store_field (rv, 0, Val_false);
+  } else
+    rv = Val_none;
+
+  CAMLreturn (rv);
+}
+
+value
+virt_builder_yajl_is_available (value unit)
+{
+  /* NB: noalloc */
+  return Val_true;
+}
+
+value
+virt_builder_yajl_tree_parse (value stringv)
+{
+  CAMLparam1 (stringv);
+  CAMLlocal1 (rv);
+  yajl_val tree;
+  char error_buf[256];
+
+  tree = yajl_tree_parse (String_val (stringv), error_buf, sizeof error_buf);
+  if (tree == NULL) {
+    char buf[256 + sizeof error_buf];
+    if (strlen (error_buf) > 0)
+      snprintf (buf, sizeof buf, "JSON parse error: %s", error_buf);
+    else
+      snprintf (buf, sizeof buf, "unknown JSON parse error");
+    caml_invalid_argument (buf);
+  }
+
+  rv = convert_yajl_value (tree, 1);
+  yajl_tree_free (tree);
+
+  CAMLreturn (rv);
+}
+
+#else
+
+value
+virt_builder_yajl_is_available (value unit)
+{
+  /* NB: noalloc */
+  return Val_false;
+}
+
+value
+virt_builder_yajl_tree_parse (value stringv)
+{
+  caml_invalid_argument ("virt-builder was compiled without yajl
support");
+}
+
+#endif
diff --git a/builder/yajl.ml b/builder/yajl.ml
new file mode 100644
index 0000000..f2d5c2b
--- /dev/null
+++ b/builder/yajl.ml
@@ -0,0 +1,30 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type yajl_val +| Yajl_null
+| Yajl_string of string
+| Yajl_number of int64
+| Yajl_double of float
+| Yajl_object of (string * yajl_val) array
+| Yajl_array of yajl_val array
+| Yajl_bool of bool
+
+external yajl_is_available : unit -> bool =
"virt_builder_yajl_is_available" "noalloc"
+
+external yajl_tree_parse : string -> yajl_val =
"virt_builder_yajl_tree_parse"
diff --git a/builder/yajl.mli b/builder/yajl.mli
new file mode 100644
index 0000000..aaa9389
--- /dev/null
+++ b/builder/yajl.mli
@@ -0,0 +1,33 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type yajl_val +| Yajl_null
+| Yajl_string of string
+| Yajl_number of int64
+| Yajl_double of float
+| Yajl_object of (string * yajl_val) array
+| Yajl_array of yajl_val array
+| Yajl_bool of bool
+
+val yajl_is_available : unit -> bool
+(** Is YAJL built in? If not, calling any of the other yajl_*
+    functions will result in an error. *)
+
+val yajl_tree_parse : string -> yajl_val
+(** Parse the JSON string. *)
diff --git a/builder/yajl_tests.ml b/builder/yajl_tests.ml
new file mode 100644
index 0000000..344a8db
--- /dev/null
+++ b/builder/yajl_tests.ml
@@ -0,0 +1,139 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(* This file tests the Yajl module. *)
+
+open OUnit2
+open Yajl
+
+(* Utils. *)
+let assert_equal_string = assert_equal ~printer:(fun x -> x)
+let assert_equal_int = assert_equal ~printer:(fun x -> string_of_int x)
+let assert_equal_int64 = assert_equal ~printer:(fun x -> Int64.to_string x)
+let assert_equal_bool = assert_equal ~printer:(fun x -> string_of_bool x)
+
+let string_of_yajl_val_type = function
+  | Yajl_null -> "null"
+  | Yajl_string _ -> "string"
+  | Yajl_number _ -> "number"
+  | Yajl_double _ -> "float"
+  | Yajl_object _ -> "object"
+  | Yajl_array _ -> "array"
+  | Yajl_bool _ -> "bool"
+let type_mismatch_string exp value +  Printf.sprintf "value is not %s but
%s" exp (string_of_yajl_val_type value)
+
+let assert_raises_invalid_argument str +  (* Replace the Invalid_argument
string with a fixed one, just to check
+   * whether the exception has been raised.
+   *)
+  let mock = "parse_error" in
+  let wrapped_tree_parse str +    try yajl_tree_parse str
+    with Invalid_argument _ -> raise (Invalid_argument mock) in
+  assert_raises (Invalid_argument mock) (fun () -> wrapped_tree_parse str)
+let assert_raises_nested str +  let err = "too many levels of object/array
nesting" in
+  assert_raises (Invalid_argument err) (fun () -> yajl_tree_parse str)
+
+let assert_is_object value +  assert_bool
+    (type_mismatch_string "object" value)
+    (match value with | Yajl_object _ -> true | _ -> false)
+let assert_is_string exp = function
+  | Yajl_string s -> assert_equal_string exp s
+  | _ as v -> assert_failure (type_mismatch_string "string" v)
+let assert_is_number exp = function
+  | Yajl_number n -> assert_equal_int64 exp n
+  | Yajl_double d -> assert_equal_int64 exp (Int64.of_float d)
+  | _ as v -> assert_failure (type_mismatch_string "number/double"
v)
+let assert_is_array value +  assert_bool
+    (type_mismatch_string "array" value)
+    (match value with | Yajl_array _ -> true | _ -> false)
+let assert_is_bool exp = function
+  | Yajl_bool b -> assert_equal_bool exp b
+  | _ as v -> assert_failure (type_mismatch_string "bool" v)
+
+let get_object_list = function
+  | Yajl_object x -> x
+  | _ as v -> assert_failure (type_mismatch_string "object" v)
+let get_array = function
+  | Yajl_array x -> x
+  | _ as v -> assert_failure (type_mismatch_string "array" v)
+
+
+let test_tree_parse_invalid ctx +  assert_raises_invalid_argument "";
+  assert_raises_invalid_argument "invalid";
+  assert_raises_invalid_argument ":5";
+
+  (* Nested objects/arrays. *)
+  let str = "[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]" in
+  assert_raises_nested str;
+  let str =
"{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":5}}}}}}}}}}}}}}}}}}}}}"
in
+  assert_raises_nested str
+
+let test_tree_parse_basic ctx +  let value = yajl_tree_parse "{}" in
+  assert_is_object value;
+
+  let value = yajl_tree_parse "\"foo\"" in
+  assert_is_string "foo" value;
+
+  let value = yajl_tree_parse "[]" in
+  assert_is_array value
+
+let test_tree_parse_inspect ctx +  let value = yajl_tree_parse
"{\"foo\":5}" in
+  let l = get_object_list value in
+  assert_equal_int 1 (Array.length l);
+  assert_equal_string "foo" (fst (l.(0)));
+  assert_is_number 5_L (snd (l.(0)));
+
+  let value = yajl_tree_parse "[\"foo\", true]" in
+  let a = get_array value in
+  assert_equal_int 2 (Array.length a);
+  assert_is_string "foo" (a.(0));
+  assert_is_bool true (a.(1));
+
+  let value = yajl_tree_parse "{\"foo\":[false, {}, 10],
\"second\":2}" in
+  let l = get_object_list value in
+  assert_equal_int 2 (Array.length l);
+  assert_equal_string "foo" (fst (l.(0)));
+  let a = get_array (snd (l.(0))) in
+  assert_equal_int 3 (Array.length a);
+  assert_is_bool false (a.(0));
+  assert_is_object (a.(1));
+  assert_is_number 10_L (a.(2));
+  assert_equal_string "second" (fst (l.(1)));
+  assert_is_number 2_L (snd (l.(1)))
+
+(* Suites declaration. *)
+let suite +  "builder Yajl" >:::
+    [
+      "tree_parse.invalid" >:: test_tree_parse_invalid;
+      "tree_parse.basic" >:: test_tree_parse_basic;
+      "tree_parse.inspect" >:: test_tree_parse_inspect;
+    ]
+
+let () +  if not (yajl_is_available ()) then
+    exit 77;
+  run_test_tt_main suite
diff --git a/po/POTFILES b/po/POTFILES
index 6a0a3fc..bb68183 100644
--- a/po/POTFILES
+++ b/po/POTFILES
@@ -6,6 +6,7 @@ builder/index-struct.c
 builder/index-validate.c
 builder/pxzcat-c.c
 builder/setlocale-c.c
+builder/yajl-c.c
 cat/cat.c
 cat/filesystems.c
 cat/log.c
diff --git a/po/POTFILES-ml b/po/POTFILES-ml
index 7933c8e..ff08a53 100644
--- a/po/POTFILES-ml
+++ b/po/POTFILES-ml
@@ -14,6 +14,8 @@ builder/setlocale.ml
 builder/sigchecker.ml
 builder/sources.ml
 builder/utils.ml
+builder/yajl.ml
+builder/yajl_tests.ml
 customize/crypt.ml
 customize/customize_cmdline.ml
 customize/customize_main.ml
-- 
2.1.0

This way it can be used in automake conditionals.
---
 configure.ac | 1 +
 1 file changed, 1 insertion(+)

diff --git a/configure.ac b/configure.ac
index 739cf06..2ce2456 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1072,6 +1072,7 @@ PKG_CHECK_MODULES([YAJL], [yajl >= 2.0.4], [
     AC_SUBST([YAJL_LIBS])
     AC_DEFINE([HAVE_YAJL],[1],[Define to 1 if you have yajl.])
 ],[AC_MSG_WARN([yajl not found, some features will be disabled])])
+AM_CONDITIONAL([HAVE_YAJL],[test "x$YAJL_LIBS" != "x"])
 
 dnl Check for C++ (optional, we just use this to test the header works).
 AC_PROG_CXX
-- 
2.1.0

Add a new "simplestreams" repository type, and a simple parser for
fetching and reading the JSON indexes of the Simple Streams v1.0
format.

Read only datatype=image-downloads contents, and only the latest
versions of each content available as disk image (disk.img or
disk1.img).

Add a simple test, using the "released" images from the CirrOS
project.
---
 .gitignore                                         |   1 +
 builder/Makefile.am                                |   6 +
 builder/builder.ml                                 |   2 +
 builder/simplestreams_parser.ml                    | 204 ++++++++++
 builder/simplestreams_parser.mli                   |  19 +
 builder/sources.ml                                 |   9 +
 builder/sources.mli                                |   1 +
 builder/test-simplestreams/streams/v1/index.json   |  18 +
 .../v1/net.cirros-cloud:released:download.json     | 429 +++++++++++++++++++++
 .../virt-builder/repos.d/cirros.conf.in            |   3 +
 builder/test-virt-builder-list-simplestreams.sh    | 108 ++++++
 builder/virt-builder.pod                           |   7 +
 configure.ac                                       |   1 +
 po/POTFILES-ml                                     |   1 +
 14 files changed, 809 insertions(+)
 create mode 100644 builder/simplestreams_parser.ml
 create mode 100644 builder/simplestreams_parser.mli
 create mode 100644 builder/test-simplestreams/streams/v1/index.json
 create mode 100644
builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json
 create mode 100644
builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in
 create mode 100755 builder/test-virt-builder-list-simplestreams.sh

diff --git a/.gitignore b/.gitignore
index db8d0a2..a430f6e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,6 +65,7 @@ Makefile.in
 /builder/stamp-virt-builder.pod
 /builder/stamp-virt-index-validate.pod
 /builder/test-config/virt-builder/repos.d/test-index.conf
+/builder/test-simplestreams/virt-builder/repos.d/cirros.conf
 /builder/test-website/virt-builder/repos.d/libguestfs.conf
 /builder/virt-builder
 /builder/virt-builder.1
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 366b8db..e7cc444 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -49,6 +49,7 @@ SOURCES_MLI = \
 	pxzcat.mli \
 	setlocale.mli \
 	sigchecker.mli \
+	simplestreams_parser.mli \
 	sources.mli \
 	yajl.mli
 
@@ -67,6 +68,7 @@ SOURCES_ML = \
 	downloader.ml \
 	sigchecker.ml \
 	index_parser.ml \
+	simplestreams_parser.ml \
 	list_entries.ml \
 	cmdline.ml \
 	builder.ml
@@ -269,6 +271,10 @@ TESTS = \
 	test-virt-index-validate.sh
 check_PROGRAMS  
+if HAVE_YAJL
+TESTS += test-virt-builder-list-simplestreams.sh
+endif
+
 if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
diff --git a/builder/builder.ml b/builder/builder.ml
index dcfd437..7d347b3 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -182,6 +182,8 @@ let main ()            match source.Sources.format with
           | Sources.FormatNative ->
             Index_parser.get_index ~downloader ~sigchecker source
+          | Sources.FormatSimpleStreams ->
+            Simplestreams_parser.get_index ~downloader ~sigchecker source
       ) sources
     ) in
   let index = remove_duplicates index in
diff --git a/builder/simplestreams_parser.ml b/builder/simplestreams_parser.ml
new file mode 100644
index 0000000..13e0b5d
--- /dev/null
+++ b/builder/simplestreams_parser.ml
@@ -0,0 +1,204 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Yajl
+open Utils
+
+open Printf
+
+let ensure_trailing_slash str +  if String.length str > 0 &&
str.[String.length str - 1] <> '/' then str ^ "/"
+  else str
+
+let object_find_optional key = function
+  | Yajl_object o ->
+    (match List.filter (fun (k, _) -> k = key) (Array.to_list o) with
+    | [(k, v)] -> Some v
+    | [] -> None
+    | _ -> error (f_"more than value for the key '%s'")
key)
+  | _ -> error (f_"the value of the key '%s' is not an
object") key
+
+let object_find key yv +  match object_find_optional key yv with
+  | None -> error (f_"missing value for the key '%s'") key
+  | Some v -> v
+
+let object_get_string key yv +  match object_find key yv with
+  | Yajl_string s -> s
+  | _ -> error (f_"the value for the key '%s' is not a
string") key
+
+let object_find_object key yv +  match object_find key yv with
+  | Yajl_object _ as o -> o
+  | _ -> error (f_"the value for the key '%s' is not an
object") key
+
+let object_find_objects fn = function
+  | Yajl_object o -> filter_map fn (Array.to_list o)
+  | _ -> error (f_"the value is not an object")
+
+let object_get_object key yv +  match object_find_object key yv with
+  | Yajl_object o -> o
+  | _ -> assert false (* object_find_object already errors out. *)
+
+let object_get_number key yv +  match object_find key yv with
+  | Yajl_number n -> n
+  | Yajl_double d -> Int64.of_float d
+  | _ -> error (f_"the value for the key '%s' is not an
integer") key
+
+let objects_get_string key yvs +  let rec loop = function
+    | [] -> None
+    | x :: xs ->
+      (match object_find_optional key x with
+      | Some (Yajl_string s) -> Some s
+      | Some _ -> error (f_"the value for key '%s' is not a
string as expected") key
+      | None -> loop xs
+      )
+  in
+  match loop yvs with
+  | Some s -> s
+  | None -> error (f_"the key '%s' was not found in a list of
objects") key
+
+let get_index ~downloader ~sigchecker
+  { Sources.uri = uri; proxy = proxy } +
+  let uri = ensure_trailing_slash uri in
+
+  let download_and_parse uri +    let tmpfile, delete_tmpfile =
Downloader.download downloader ~proxy uri in
+    if delete_tmpfile then
+      unlink_on_exit tmpfile;
+    let file +      if Sigchecker.verifying_signatures sigchecker then (
+        let tmpunsigned +          Sigchecker.verify_and_remove_signature
sigchecker tmpfile in
+        match tmpunsigned with
+        | None -> assert false (* only when not verifying signatures *)
+        | Some f -> f
+      ) else
+        tmpfile in
+    yajl_tree_parse (read_whole_file file) in
+
+  let downloads +    let uri_index +      if Sigchecker.verifying_signatures
sigchecker then
+        uri ^ "streams/v1/index.sjson"
+      else
+        uri ^ "streams/v1/index.json" in
+    let tree = download_and_parse uri_index in
+
+    let format = object_get_string "format" tree in
+    if format <> "index:1.0" then
+      error (f_"%s is not a Simple Streams (index) v1.0 JSON file (format:
%s)")
+        uri format;
+
+    let index = Array.to_list (object_get_object "index" tree) in
+    filter_map (
+      fun (_, desc) ->
+        let format = object_get_string "format" desc in
+        let datatype = object_get_string "datatype" desc in
+        match format, datatype with
+        | "products:1.0", "image-downloads" ->
+          Some (object_get_string "path" desc)
+        | _ -> None
+    ) index in
+
+  let scan_product_list path +    let tree = download_and_parse (uri ^ path) in
+
+    let format = object_get_string "format" tree in
+    if format <> "products:1.0" then
+      error (f_"%s is not a Simple Streams (products) v1.0 JSON file
(format: %s)")
+        uri format;
+
+    let products_node = object_get_object "products" tree in
+
+    let products = Array.to_list products_node in
+    filter_map (
+      fun (prod, prod_desc) ->
+        let arch = object_get_string "arch" prod_desc in
+        let prods = Array.to_list (object_get_object "versions"
prod_desc) in
+        let prods = filter_map (
+          fun (rel, rel_desc) ->
+            let pubname = objects_get_string "pubname" [rel_desc;
prod_desc] in
+            let items = object_find_object "items" rel_desc in
+            let disk_items = object_find_objects (
+              function
+              | (("disk.img"|"disk1.img"), v) -> Some v
+              | _ -> None
+            ) items in
+            (match disk_items with
+            | [] -> None
+            | disk_item :: _ ->
+              let printable_name = Some pubname in
+              let file_uri = uri ^ (object_get_string "path"
disk_item) in
+              let checksums +                let checksums =
object_find_objects (
+                  function
+                  | ("sha256", Yajl_string c) -> Some
(Checksums.SHA256 c)
+                  | ("sha512", Yajl_string c) -> Some
(Checksums.SHA512 c)
+                  | _ -> None
+                ) disk_item in
+                match checksums with
+                | [] -> None
+                | x -> Some x in
+              let revision = Rev_string rel in
+              let size = object_get_number "size" disk_item in
+              let aliases = Some [pubname;] in
+
+              let entry = { Index.printable_name = printable_name;
+                            osinfo = None;
+                            file_uri = file_uri;
+                            arch = arch;
+                            signature_uri = None;
+                            checksums = checksums;
+                            revision = revision;
+                            format = None;
+                            size = size;
+                            compressed_size = None;
+                            expand = None;
+                            lvexpand = None;
+                            notes = [];
+                            hidden = false;
+                            aliases = aliases;
+                            sigchecker = sigchecker;
+                            proxy = proxy; } in
+              Some (rel, (prod, entry))
+            )
+        ) prods in
+        (* Select the disk image with the bigger version (i.e. usually
+         * the most recent one. *)
+        let reverse_revision_compare (rev1, _) (rev2, _) = compare rev2 rev1 in
+        let prods = List.sort reverse_revision_compare prods in
+        match prods with
+        | [] -> None
+        | (_, entry) :: _ -> Some entry
+    ) products in
+
+  let entries = List.flatten (List.map scan_product_list downloads) in
+  if verbose () then (
+    printf "simplestreams tree (%s) after parsing:\n" uri;
+    List.iter (Index.print_entry Pervasives.stdout) entries
+  );
+  entries
diff --git a/builder/simplestreams_parser.mli b/builder/simplestreams_parser.mli
new file mode 100644
index 0000000..a4b91ba
--- /dev/null
+++ b/builder/simplestreams_parser.mli
@@ -0,0 +1,19 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t ->
Sources.source -> Index.index
diff --git a/builder/sources.ml b/builder/sources.ml
index b21e8fc..149db6f 100644
--- a/builder/sources.ml
+++ b/builder/sources.ml
@@ -31,6 +31,7 @@ type source = {
 }
 and source_format  | FormatNative
+| FormatSimpleStreams
 
 module StringSet = Set.Make (String)
 
@@ -82,6 +83,14 @@ let parse_conf file            try
             (match (List.assoc ("format", None) fields) with
             | "native" | "" -> FormatNative
+            | "simplestreams" as fmt ->
+              if not (Yajl.yajl_is_available ()) then (
+                if verbose () then (
+                  eprintf (f_"%s: repository type '%s' not
supported (missing YAJL support), skipping it\n") prog fmt;
+                );
+                invalid_arg fmt
+              ) else
+                FormatSimpleStreams
             | fmt ->
               if verbose () then (
                 eprintf (f_"%s: unknown repository type '%s' in
%s, skipping it\n") prog fmt file;
diff --git a/builder/sources.mli b/builder/sources.mli
index e861310..e621a9f 100644
--- a/builder/sources.mli
+++ b/builder/sources.mli
@@ -25,5 +25,6 @@ type source = {
 }
 and source_format  | FormatNative
+| FormatSimpleStreams
 
 val read_sources : unit -> source list
diff --git a/builder/test-simplestreams/streams/v1/index.json
b/builder/test-simplestreams/streams/v1/index.json
new file mode 100644
index 0000000..3180af7
--- /dev/null
+++ b/builder/test-simplestreams/streams/v1/index.json
@@ -0,0 +1,18 @@
+{
+ "format": "index:1.0",
+ "updated": "Fri, 08 May 2015 13:20:51 +0000",
+ "index": {
+  "net.cirros-cloud:released:download": {
+   "products": [
+    "net.cirros-cloud:standard:0.3:i386",
+    "net.cirros-cloud:standard:0.3:x86_64",
+    "net.cirros-cloud:standard:0.3:powerpc",
+    "net.cirros-cloud:standard:0.3:arm"
+   ],
+   "datatype": "image-downloads",
+   "format": "products:1.0",
+   "updated": "Fri, 08 May 2015 13:20:51 +0000",
+   "path":
"streams/v1/net.cirros-cloud:released:download.json"
+  }
+ }
+}
diff --git
a/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json
b/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json
new file mode 100644
index 0000000..0fb57b4
--- /dev/null
+++
b/builder/test-simplestreams/streams/v1/net.cirros-cloud:released:download.json
@@ -0,0 +1,429 @@
+{
+ "products": {
+  "net.cirros-cloud:standard:0.3:i386": {
+   "arch": "i386",
+   "versions": {
+    "20140908": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "785c524f55bf2493dcddeedba25172f1",
+       "sha256":
"bc20b950cc20e66e0b915748ad6b1ea2360ec03f08ffe6c68dc1b4a82331ab66",
+       "size": 3204602,
+       "path": "0.3.3/cirros-0.3.3-i386-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "283c77db6fc79b2d47c585ec241e7edc",
+       "sha256":
"837cc3c79de14de62b8691c08a6913b7c0c62427870c7c1921ac43b9580748a9",
+       "size": 12268032,
+       "path": "0.3.3/cirros-0.3.3-i386-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "861ef94d90ea220ae72b84d258ff5620",
+       "sha256":
"5c7e685d42f0f1c5c70398e8b7388fc21b84dcbb4dc4a37c1c1645fe5c5ed3a4",
+       "size": 8231317,
+       "path": "0.3.3/cirros-0.3.3-i386-uec.tar.gz"
+      }
+     },
+     "version": "0.3.3",
+     "pubname": "cirros-0.3.3-i386"
+    },
+    "20150422": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "ffe02ef2b49cb34881f96e2b4c69383c",
+       "sha256":
"5760cae9b9701dc09f495c3787f1da271800dcbc040eb793ddf4eb636689148a",
+       "size": 3210378,
+       "path": "0.3.4/cirros-0.3.4-i386-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "79b4436412283bb63c2cba4ac796bcd9",
+       "sha256":
"48c279ff502ea4c5e3e09d58c9e3a2ab615452dae2ed8fe347f68c136c6a5a0b",
+       "size": 12506112,
+       "path": "0.3.4/cirros-0.3.4-i386-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "c1630b33986cc09df2e0cbca7f5c5346",
+       "sha256":
"f1cad519da6632b1d0ebad2f892534b732b5f9dd54d27f532d2963a0748f246e",
+       "size": 8241360,
+       "path": "0.3.4/cirros-0.3.4-i386-uec.tar.gz"
+      }
+     },
+     "version": "0.3.4",
+     "pubname": "cirros-0.3.4-i386"
+    },
+    "20140317": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "55023b89a6121b30603859bb80f5c8d3",
+       "sha256":
"136f8dfff85b8d509a0d0a0264aea79ae94d9b7c3ea5745fbd478ae600adeb20",
+       "size": 3201206,
+       "path": "0.3.2/cirros-0.3.2-i386-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "b0a256d40e078d66576f7ae91c296e2b",
+       "sha256":
"f0803c2d179c8a02d029239d35fc3e752cc81ad3436ea52b757e11685ca7c074",
+       "size": 12336128,
+       "path": "0.3.2/cirros-0.3.2-i386-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "dcbcc559fc3bc468d9149bd7191aaa0c",
+       "sha256":
"74ba8b67e0dce67f16a6bc5942bee90cbe31381f932aac1ba51bbc774aad5046",
+       "size": 8221995,
+       "path": "0.3.2/cirros-0.3.2-i386-uec.tar.gz"
+      }
+     },
+     "version": "0.3.2",
+     "pubname": "cirros-0.3.2-i386"
+    },
+    "20111020": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "e760bf470841f57c2c1bb426d407d169",
+       "sha256":
"cac7887628527604b65a89e8caa34096d51c2dc1acfe405c15db2fc58495142a",
+       "size": 1845928,
+       "path": "0.3.0/cirros-0.3.0-i386-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "90169ba6f09b5906a7f0755bd00bf2c3",
+       "sha256":
"3309675d0d409128b1c2651d576bc8092ca9ab93e15f3d3aa458f40947569b61",
+       "size": 9159168,
+       "path": "0.3.0/cirros-0.3.0-i386-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "115ca6afa47089dc083c0dc9f9b7ff03",
+       "sha256":
"b57e0acb32852f89734ff11a511ae0897e8cecb41882d03551649289b6854a1b",
+       "size": 6596586,
+       "path": "0.3.0/cirros-0.3.0-i386-uec.tar.gz"
+      }
+     },
+     "version": "0.3.0",
+     "pubname": "cirros-0.3.0-i386"
+    },
+    "20130207": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "fc404d9fc9dc5f0eb33ebaa03920a046",
+       "sha256":
"70a8c9c175589f7ac7054c6151cf2bb7eb9e210cefbe310446df2fb1a436b504",
+       "size": 3191593,
+       "path": "0.3.1/cirros-0.3.1-i386-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "6ba617eafc992e33e7c141c679225e53",
+       "sha256":
"b8aa1ce5d11939eaa01205fc31348532a31b82790921d45ceb397fbe76492787",
+       "size": 12251136,
+       "path": "0.3.1/cirros-0.3.1-i386-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "52845de5142e58faf211e135d2b45721",
+       "sha256":
"88dda2e505b862a95d0e0044013addcaa3200e602150c9d73e32c2e29345d6f3",
+       "size": 8197543,
+       "path": "0.3.1/cirros-0.3.1-i386-uec.tar.gz"
+      }
+     },
+     "version": "0.3.1",
+     "pubname": "cirros-0.3.1-i386"
+    }
+   },
+   "stream": "released"
+  },
+  "net.cirros-cloud:standard:0.3:x86_64": {
+   "arch": "x86_64",
+   "versions": {
+    "20140908": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "600363faf89daa6adea87e2dd77263b4",
+       "sha256":
"66255c2f9404484996e646bb825fd2f1fd9d66583fecd3e9e0789376cf055fd4",
+       "size": 3544884,
+       "path": "0.3.3/cirros-0.3.3-x86_64-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "133eae9fb1c98f45894a4e60d8736619",
+       "sha256":
"f11286e2bd317ee1a1d0469a6b182b33bda4af6f35ba224ca49d75752c81e20a",
+       "size": 13200896,
+       "path": "0.3.3/cirros-0.3.3-x86_64-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "4f3371e52d60e5d47a6fbdd7d6a33973",
+       "sha256":
"6bd27854862c30e25f0312defb9e86f9176b503751f040e323ab83322aa07dae",
+       "size": 8668951,
+       "path": "0.3.3/cirros-0.3.3-x86_64-uec.tar.gz"
+      }
+     },
+     "version": "0.3.3",
+     "pubname": "cirros-0.3.3-x86_64"
+    },
+    "20150422": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "6554ba2d15401c8caa90212ace985593",
+       "sha256":
"e8172c603fad47f4c95e67bd2751c2977e07164c9ebdfcf3f9c1d7ff598ed217",
+       "size": 3555536,
+       "path": "0.3.4/cirros-0.3.4-x86_64-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "ee1eca47dc88f4879d8a229cc70a07c6",
+       "sha256":
"34987d0d5702f8813f3ff9efe90e9e39e6926ec78658763580a79face67f3394",
+       "size": 13287936,
+       "path": "0.3.4/cirros-0.3.4-x86_64-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "067d511efcc4acd034f7d64b716273bf",
+       "sha256":
"95e77c7deaf0f515f959ffe329918d5dd23e417503d1d45e926a888853c90710",
+       "size": 8683894,
+       "path": "0.3.4/cirros-0.3.4-x86_64-uec.tar.gz"
+      }
+     },
+     "version": "0.3.4",
+     "pubname": "cirros-0.3.4-x86_64"
+    },
+    "20140317": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "174a08e827f22b20ebd774f90f1c9181",
+       "sha256":
"03eff2a3603164fa02c00ca76fdcbd611b2ae8a6a80406753ec35eb275648d2e",
+       "size": 3538286,
+       "path": "0.3.2/cirros-0.3.2-x86_64-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "64d7c1cd2b6f60c92c14662941cb7913",
+       "sha256":
"a2ca56aeded5a5bcaa6104fb14ec07b1ceb65222e2890bef8a89b8d2da729ad5",
+       "size": 13167616,
+       "path": "0.3.2/cirros-0.3.2-x86_64-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "b369559e3ffa9208b4eb2224c949f579",
+       "sha256":
"f462729fc2f071081dbc55932e07437e265263ef2e688306c353a1f152162b03",
+       "size": 8655821,
+       "path": "0.3.2/cirros-0.3.2-x86_64-uec.tar.gz"
+      }
+     },
+     "version": "0.3.2",
+     "pubname": "cirros-0.3.2-x86_64"
+    },
+    "20111020": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "1a480e5150db4d93be2aa3c9ced94fa1",
+       "sha256":
"0e78796a30641dd7184f752c302a87d66f1eba9985c876911e4f26b4d8ba4a88",
+       "size": 2115217,
+       "path": "0.3.0/cirros-0.3.0-x86_64-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "50bdc35edb03a38d91b1b071afb20a3c",
+       "sha256":
"648782e9287288630250d07531fed9944ecc3986764a6664f0bf6c050ec06afd",
+       "size": 9761280,
+       "path": "0.3.0/cirros-0.3.0-x86_64-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "f56d3cffa47b7d209d2b6905628f07b9",
+       "sha256":
"043a3e090a5d76d23758a3919fcaff93f77ce7b97594d9d10fc8d00e85f83191",
+       "size": 6957349,
+       "path": "0.3.0/cirros-0.3.0-x86_64-uec.tar.gz"
+      }
+     },
+     "version": "0.3.0",
+     "pubname": "cirros-0.3.0-x86_64"
+    },
+    "20130207": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "38252f1a49fec0ebedebc820854497e0",
+       "sha256":
"a086fcf2758468972d45957dc78ec6317c06f356930dbbc6cad6a8d1855f135e",
+       "size": 3534564,
+       "path": "0.3.1/cirros-0.3.1-x86_64-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "d972013792949d0d3ba628fbe8685bce",
+       "sha256":
"e01302fb2d2b13ae65226a0300335172e4487bbe60bb1e5c8b0843a25f126d34",
+       "size": 13147648,
+       "path": "0.3.1/cirros-0.3.1-x86_64-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "e1849016cb71a00808093b7bf986f36a",
+       "sha256":
"51eb03b83123a68d4f866c7c15b195204e62db9e33475509a38b79b3122cde38",
+       "size": 8633554,
+       "path": "0.3.1/cirros-0.3.1-x86_64-uec.tar.gz"
+      }
+     },
+     "version": "0.3.1",
+     "pubname": "cirros-0.3.1-x86_64"
+    }
+   },
+   "stream": "released"
+  },
+  "net.cirros-cloud:standard:0.3:powerpc": {
+   "arch": "powerpc",
+   "versions": {
+    "20150422": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "7e3a147b5665a1e14fe6d0ffef9ca410",
+       "sha256":
"b1ab58ebb2a3f253d15cfa13f0cd0de883ce6e80f42ce2e3889e880423e95462",
+       "size": 3603000,
+       "path": "0.3.4/cirros-0.3.4-powerpc-lxc.tar.gz"
+      },
+      "disk.img": {
+       "ftype": "disk.img",
+       "md5": "453b21916c47c6ff8c615f8a5f7b76d2",
+       "sha256":
"5445d77104d19ccdc2dbc943a3d0735c0dadd1227466a3f3e109c19671065f18",
+       "size": 17145856,
+       "path": "0.3.4/cirros-0.3.4-powerpc-disk.img"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "2f8a48928a6b0b2c9afbcb7255ff2aec",
+       "sha256":
"80444c3ac8277d87eed0963567206ea5b009ecf4fdbc71fe8c4df020b42e3215",
+       "size": 12032417,
+       "path": "0.3.4/cirros-0.3.4-powerpc-uec.tar.gz"
+      }
+     }
+    }
+   },
+   "version": "0.3.4",
+   "stream": "released",
+   "pubname": "cirros-0.3.4-powerpc"
+  },
+  "net.cirros-cloud:standard:0.3:arm": {
+   "arch": "arm",
+   "versions": {
+    "20140908": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "7f2fe450d9a0b1688f2a4df6748690f5",
+       "sha256":
"c839db36d5c96651d1dbcfc8a553678b010c7bb49416ac3099282858003fa021",
+       "size": 3482025,
+       "path": "0.3.3/cirros-0.3.3-arm-lxc.tar.gz"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "0437419d2371771b7406fade2d6db7d1",
+       "sha256":
"4a0987818f3a834d9907b93729beb79426e58be9d46b3c207241bc4b1ceec52f",
+       "size": 7351177,
+       "path": "0.3.3/cirros-0.3.3-arm-uec.tar.gz"
+      }
+     },
+     "version": "0.3.3",
+     "pubname": "cirros-0.3.3-arm"
+    },
+    "20150422": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "5fd856d158dc6787264a1b4236126aab",
+       "sha256":
"06c780e8c7166e16154bd0d3a6d1b0031e219269e990c5b8864a82925285486c",
+       "size": 3489736,
+       "path": "0.3.4/cirros-0.3.4-arm-lxc.tar.gz"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "1a2aea370f9f2d95d837f6b84bef658d",
+       "sha256":
"6dd3bad904671aee13317c187d933b3efb1ac39a0d81fd5ac9a1fc694e3cb7ff",
+       "size": 7357906,
+       "path": "0.3.4/cirros-0.3.4-arm-uec.tar.gz"
+      }
+     },
+     "version": "0.3.4",
+     "pubname": "cirros-0.3.4-arm"
+    },
+    "20140317": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "bfb2ecb203cf27f4786b43a558b1ffe8",
+       "sha256":
"a916b3d268793977001e2f09638e4a3e3953cc51d0d53d0bfb8c14cfa7105e86",
+       "size": 3474403,
+       "path": "0.3.2/cirros-0.3.2-arm-lxc.tar.gz"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "171719bfd4b3ec613f5fac321e4216de",
+       "sha256":
"7f74826638e153ec58230bae9eb21805f875ed12eab299140f978e45597cb5cd",
+       "size": 7340357,
+       "path": "0.3.2/cirros-0.3.2-arm-uec.tar.gz"
+      }
+     },
+     "version": "0.3.2",
+     "pubname": "cirros-0.3.2-arm"
+    },
+    "20111020": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "91add49e56cbe6b5004015a4d2f51dbc",
+       "sha256":
"fcd3723c956a1c232730dc28513b466657cbe984232ba2fcc30a4e1f55aa91e9",
+       "size": 2043822,
+       "path": "0.3.0/cirros-0.3.0-arm-lxc.tar.gz"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "c31e05f7829ad45f9d9995c35d232769",
+       "sha256":
"b871823406f818430f57744333b1bb17ce0047e551a316f316641f1bd70d9152",
+       "size": 5761642,
+       "path": "0.3.0/cirros-0.3.0-arm-uec.tar.gz"
+      }
+     },
+     "version": "0.3.0",
+     "pubname": "cirros-0.3.0-arm"
+    },
+    "20130207": {
+     "items": {
+      "lxc.tar.gz": {
+       "ftype": "lxc.tar.gz",
+       "md5": "7ddea367ecb7ecb91554e18bed7c71bd",
+       "sha256":
"2060e59e642b3b2bdf6e34aba3ed15f468bc6f9a8417fc196d01d29b2075493e",
+       "size": 3466149,
+       "path": "0.3.1/cirros-0.3.1-arm-lxc.tar.gz"
+      },
+      "uec.tar.gz": {
+       "ftype": "uec.tar.gz",
+       "md5": "d04e6f26aed123bba2c096581b269e7f",
+       "sha256":
"09dcd3ea6f1d48b3519232973e4dc00fc5e73cbea974cda6b5f7cfa380c6b428",
+       "size": 7314471,
+       "path": "0.3.1/cirros-0.3.1-arm-uec.tar.gz"
+      }
+     },
+     "version": "0.3.1",
+     "pubname": "cirros-0.3.1-arm"
+    }
+   },
+   "stream": "released"
+  }
+ },
+ "datatype": "image-downloads",
+ "format": "products:1.0",
+ "updated": "Fri, 08 May 2015 13:20:51 +0000",
+ "content_id": "net.cirros-cloud:released:download"
+}
diff --git a/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in
b/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in
new file mode 100644
index 0000000..a6f2765
--- /dev/null
+++ b/builder/test-simplestreams/virt-builder/repos.d/cirros.conf.in
@@ -0,0 +1,3 @@
+[test-cirros]
+uri=file://@abs_top_builddir@/builder/test-simplestreams
+format=simplestreams
diff --git a/builder/test-virt-builder-list-simplestreams.sh
b/builder/test-virt-builder-list-simplestreams.sh
new file mode 100755
index 0000000..24cf44c
--- /dev/null
+++ b/builder/test-virt-builder-list-simplestreams.sh
@@ -0,0 +1,108 @@
+#!/bin/bash -
+# libguestfs virt-builder test script
+# Copyright (C) 2015 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+abs_builddir=$(pwd)
+
+export XDG_CONFIG_HOME+export
XDG_CONFIG_DIRS="$abs_builddir/test-simplestreams"
+
+short_list=$($VG virt-builder --no-check-signature --no-cache --list)
+
+if [ "$short_list" != "net.cirros-cloud:standard:0.3:i386 i386  
cirros-0.3.4-i386
+net.cirros-cloud:standard:0.3:x86_64 x86_64     cirros-0.3.4-x86_64
+net.cirros-cloud:standard:0.3:powerpc powerpc    cirros-0.3.4-powerpc" ];
then
+    echo "$0: unexpected --list output:"
+    echo "$short_list"
+    exit 1
+fi
+
+long_list=$(virt-builder --no-check-signature --no-cache --list --long)
+
+if [ "$long_list" != "Source URI:
file://$abs_builddir/test-simplestreams
+
+os-version:              net.cirros-cloud:standard:0.3:i386
+Full name:               cirros-0.3.4-i386
+Architecture:            i386
+Minimum/default size:    11.9M
+Aliases:                 cirros-0.3.4-i386
+
+os-version:              net.cirros-cloud:standard:0.3:x86_64
+Full name:               cirros-0.3.4-x86_64
+Architecture:            x86_64
+Minimum/default size:    12.7M
+Aliases:                 cirros-0.3.4-x86_64
+
+os-version:              net.cirros-cloud:standard:0.3:powerpc
+Full name:               cirros-0.3.4-powerpc
+Architecture:            powerpc
+Minimum/default size:    16.4M
+Aliases:                 cirros-0.3.4-powerpc" ]; then
+    echo "$0: unexpected --list --long output:"
+    echo "$long_list"
+    exit 1
+fi
+
+json_list=$(virt-builder --no-check-signature --no-cache --list --list-format
json)
+
+if [ "$json_list" != "{
+  \"version\": 1,
+  \"sources\": [
+    {
+      \"uri\": \"file://$abs_builddir/test-simplestreams\"
+    }
+  ],
+  \"templates\": [
+    {
+      \"os-version\":
\"net.cirros-cloud:standard:0.3:i386\",
+      \"full-name\": \"cirros-0.3.4-i386\",
+      \"arch\": \"i386\",
+      \"size\": 12506112,
+      \"aliases\": [
+        \"cirros-0.3.4-i386\"
+      ],
+      \"hidden\": false
+    },
+    {
+      \"os-version\":
\"net.cirros-cloud:standard:0.3:x86_64\",
+      \"full-name\": \"cirros-0.3.4-x86_64\",
+      \"arch\": \"x86_64\",
+      \"size\": 13287936,
+      \"aliases\": [
+        \"cirros-0.3.4-x86_64\"
+      ],
+      \"hidden\": false
+    },
+    {
+      \"os-version\":
\"net.cirros-cloud:standard:0.3:powerpc\",
+      \"full-name\": \"cirros-0.3.4-powerpc\",
+      \"arch\": \"powerpc\",
+      \"size\": 17145856,
+      \"aliases\": [
+        \"cirros-0.3.4-powerpc\"
+      ],
+      \"hidden\": false
+    }
+  ]
+}" ]; then
+    echo "$0: unexpected --list --format json output:"
+    echo "$json_list"
+    exit 1
+fi
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 710f006..67e017f 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1183,6 +1183,13 @@ The possible values are:
 The native format of the C<virt-builder> repository.  See also
 L</Creating and signing the index file> below.
 
+=item B<simplestreams>
+
+The URI represents the root of a Simple Streams v1.0 tree of metadata.
+
+For more information about Simple Streams, see also
+L<https://launchpad.net/simplestreams>.
+
 =back
 
 If not present, the assumed value is C<native>.
diff --git a/configure.ac b/configure.ac
index 2ce2456..e057660 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1724,6 +1724,7 @@ AC_CONFIG_FILES([Makefile
                  builder/Makefile
                  builder/libguestfs.conf
                  builder/test-config/virt-builder/repos.d/test-index.conf
+                 builder/test-simplestreams/virt-builder/repos.d/cirros.conf
                  builder/test-website/virt-builder/repos.d/libguestfs.conf
                  builder/website/Makefile
                  cat/Makefile
diff --git a/po/POTFILES-ml b/po/POTFILES-ml
index ff08a53..3f0038e 100644
--- a/po/POTFILES-ml
+++ b/po/POTFILES-ml
@@ -12,6 +12,7 @@ builder/paths.ml
 builder/pxzcat.ml
 builder/setlocale.ml
 builder/sigchecker.ml
+builder/simplestreams_parser.ml
 builder/sources.ml
 builder/utils.ml
 builder/yajl.ml
-- 
2.1.0

Looks good, ACK series.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine.  Supports Linux and Windows.
http://people.redhat.com/~rjones/virt-df/