[Libguestfs] [PATCH 10/10] builder: support Simple Streams v1.0 as index metadata

Pino Toscano
Tue, 28 Jul 2015 11:16:04 -0000

Add a simple YAJL<->OCaml bridge to expose the JSON parsing function,
and use it to parse the JSON indexes of the Simple Streams format.

Read only datatype=image-downloads contents, and only the latest
versions of each content available as disk image (disk.img or
disk1.img).
---
 builder/Makefile.am              |  13 ++-
 builder/builder.ml               |   2 +
 builder/simplestreams_parser.ml  | 207 +++++++++++++++++++++++++++++++++++++++
 builder/simplestreams_parser.mli |  19 ++++
 builder/sources.ml               |   9 ++
 builder/sources.mli              |   1 +
 builder/virt-builder.pod         |   7 ++
 builder/yajl-c.c                 | 141 ++++++++++++++++++++++++++
 builder/yajl.ml                  |  30 ++++++
 builder/yajl.mli                 |  33 +++++++
 po/POTFILES                      |   1 +
 po/POTFILES-ml                   |   2 +
 12 files changed, 462 insertions(+), 3 deletions(-)
 create mode 100644 builder/simplestreams_parser.ml
 create mode 100644 builder/simplestreams_parser.mli
 create mode 100644 builder/yajl-c.c
 create mode 100644 builder/yajl.ml
 create mode 100644 builder/yajl.mli

diff --git a/builder/Makefile.am b/builder/Makefile.am
index 597b943..98a444f 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -48,7 +48,9 @@ SOURCES_MLI = \
 	pxzcat.mli \
 	setlocale.mli \
 	sigchecker.mli \
-	sources.mli
+	simplestreams_parser.mli \
+	sources.mli \
+	yajl.mli

 SOURCES_ML = \
 	utils.ml \
@@ -57,6 +59,7 @@ SOURCES_ML = \
 	checksums.ml \
 	index.ml \
 	ini_reader.ml \
+	yajl.ml \
 	paths.ml \
 	languages.ml \
 	cache.ml \
@@ -64,6 +67,7 @@ SOURCES_ML = \
 	downloader.ml \
 	sigchecker.ml \
 	index_parser.ml \
+	simplestreams_parser.ml \
 	list_entries.ml \
 	cmdline.ml \
 	builder.ml
@@ -81,7 +85,8 @@ SOURCES_C = \
 	index-parse.c \
 	index-parser-c.c \
 	pxzcat-c.c \
-	setlocale-c.c
+	setlocale-c.c \
+	yajl-c.c

 man_MANS  noinst_DATA @@ -106,7 +111,8 @@ virt_builder_CFLAGS = \
 	-Wno-unused-macros \
 	$(LIBLZMA_CFLAGS) \
 	$(LIBTINFO_CFLAGS) \
-	$(LIBXML2_CFLAGS)
+	$(LIBXML2_CFLAGS) \
+	$(YAJL_CFLAGS)

 BOBJECTS = \
 	$(top_builddir)/mllib/libdir.cmo \
@@ -156,6 +162,7 @@ OCAMLCLIBS = \
 	$(LIBCRYPT_LIBS) \
 	$(LIBLZMA_LIBS) \
 	$(LIBXML2_LIBS) \
+	$(YAJL_LIBS) \
 	$(LIBINTL) \
 	-lgnu

diff --git a/builder/builder.ml b/builder/builder.ml
index a30dbd1..1e6a426 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -175,6 +175,8 @@ let main ()            match source.Sources.format with
           | Sources.FormatNative ->
             Index_parser.get_index ~downloader ~sigchecker source
+          | Sources.FormatSimpleStreams ->
+            Simplestreams_parser.get_index ~downloader ~sigchecker source
       ) sources
     ) in
   let index = remove_duplicates index in
diff --git a/builder/simplestreams_parser.ml b/builder/simplestreams_parser.ml
new file mode 100644
index 0000000..5f8e725
--- /dev/null
+++ b/builder/simplestreams_parser.ml
@@ -0,0 +1,207 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Yajl
+
+open Printf
+
+let ensure_trailing_slash str +  if String.length str > 0 &&
str.[String.length str - 1] <> '/' then str ^ "/"
+  else str
+
+let object_find_optional key = function
+  | Yajl_object o ->
+    (match List.filter (fun (k, _) -> k = key) (Array.to_list o) with
+    | [(k, v)] -> Some v
+    | [] -> None
+    | _ -> error "more than one '%s' node" key)
+  | _ -> error "value for key '%s' is not an object" key
+
+let object_find key yv +  (match object_find_optional key yv with
+  | None -> error "missing '%s' node" key
+  | Some v -> v
+  )
+
+let object_get_string key yv +  (match object_find key yv with
+  | Yajl_string s -> s
+  | _ -> error "key '%s' is not a string" key
+  )
+
+let object_get_string_optional key yv +  (match object_find_optional key yv
with
+  | None -> None
+  | Some (Yajl_string s) -> Some s
+  | Some _ -> error "key '%s' is not a string" key
+  )
+
+let object_find_object key yv +  (match object_find key yv with
+  | Yajl_object _ as o -> o
+  | _ -> error "key '%s' is not an object" key
+  )
+
+let object_find_object_optional key yv +  (match object_find_optional key yv
with
+  | None -> None
+  | Some (Yajl_object _ as o) -> Some o
+  | Some _ -> error "key '%s' is not an object" key
+  )
+
+let object_find_objects fn = function
+  | Yajl_object o -> filter_map fn (Array.to_list o)
+  | _ -> error "value is not an object"
+
+let object_get_object key yv +  (match object_find_object key yv with
+  | Yajl_object o -> o
+  | _ -> assert false (* object_find_object already errors out. *)
+  )
+
+let object_get_number key yv +  (match object_find key yv with
+  | Yajl_number n -> n
+  | Yajl_double d -> Int64.of_float d
+  | _ -> error "key '%s' is not an integer" key
+  )
+
+let get_index ~downloader ~sigchecker
+  { Sources.uri = uri; proxy = proxy } +
+  let uri = ensure_trailing_slash uri in
+
+  let download_and_parse uri +    let tmpfile, delete_tmpfile =
Downloader.download downloader ~proxy uri in
+    if delete_tmpfile then
+      unlink_on_exit tmpfile;
+    let file +      if Sigchecker.verifying_signatures sigchecker then (
+        let tmpunsigned = Sigchecker.verify_and_remove_signature sigchecker
tmpfile in
+        match tmpunsigned with
+        | None -> assert false (* only when not verifying signatures *)
+        | Some f -> f
+      ) else
+        tmpfile in
+    yajl_tree_parse (read_whole_file file) in
+
+  let downloads +    let uri_index +      if Sigchecker.verifying_signatures
sigchecker then
+        uri ^ "streams/v1/index.sjson"
+      else
+        uri ^ "streams/v1/index.json" in
+    let tree = download_and_parse uri_index in
+
+    let format = object_get_string "format" tree in
+    if format <> "index:1.0" then
+      error (f_"%s does not point to a Simple Streams (index) v1.0 JSON
file") uri;
+
+    let index = Array.to_list (object_get_object "index" tree) in
+    filter_map (
+      fun (_, desc) ->
+        let datatype = object_get_string "datatype" desc in
+        match datatype with
+        | "image-downloads" -> Some (object_get_string
"path" desc)
+        | _ -> None
+    ) index in
+
+  let scan_product_list path +    let tree = download_and_parse (uri ^ path) in
+
+    let format = object_get_string "format" tree in
+    if format <> "products:1.0" then
+      error (f_"%s does not point to a Simple Streams (products) v1.0 JSON
file") uri;
+
+    let products_node = object_get_object "products" tree in
+
+    let products = Array.to_list products_node in
+    filter_map (
+      fun (prod, prod_desc) ->
+        let arch +          match object_get_string "arch" prod_desc
with
+          | "amd64" -> "x86_64"
+          | a -> a in
+        let prods = Array.to_list (object_get_object "versions"
prod_desc) in
+        let prods = filter_map (
+          fun (rel, rel_desc) ->
+            let pubname +              match object_get_string_optional
"pubname" rel_desc with
+              | Some p -> p
+              | None -> object_get_string "pubname" prod_desc in
+            let items = object_find_object "items" rel_desc in
+            let disk_items = object_find_objects (
+              function
+              | (("disk.img"|"disk1.img"), v) -> Some v
+              | _ -> None
+            ) items in
+            (match disk_items with
+            | [] -> None
+            | disk_item :: _ ->
+              let disk_path = object_get_string "path" disk_item in
+              let disk_size = object_get_number "size" disk_item in
+              let checksums = object_find_objects (
+                function
+                | ("sha256", Yajl_string c) -> Some
(Checksums.SHA256 c)
+                | ("sha512", Yajl_string c) -> Some
(Checksums.SHA512 c)
+                | _ -> None
+              ) disk_item in
+              let checksums +                match checksums with
+                | [] -> None
+                | x -> Some x in
+              let entry = {
+                Index.printable_name = Some pubname;
+                osinfo = None;
+                file_uri = uri ^ disk_path;
+                arch = arch;
+                signature_uri = None;
+                checksums = checksums;
+                revision = 0;   (* XXX handle revisions as strings *)
+                format = None;
+                size = disk_size;
+                compressed_size = None;
+                expand = None;
+                lvexpand = None;
+                notes = [];
+                hidden = false;
+                aliases = Some [pubname;];
+                sigchecker = sigchecker;
+                proxy = proxy;
+              } in
+              Some (rel, (prod, entry))
+            )
+        ) prods in
+        (* Select the disk image with the bigger version (i.e. usually
+         * the most recent one. *)
+        let reverse_revision_compare (rev1, _) (rev2, _) = compare rev2 rev1 in
+        let prods = List.sort reverse_revision_compare prods in
+        match prods with
+        | [] -> None
+        | (_, entry) :: _ -> Some entry
+    ) products in
+
+  let entries = List.flatten (List.map scan_product_list downloads) in
+  if verbose () then (
+    printf "simplestreams tree (%s) after parsing:\n" uri;
+    List.iter (Index.print_entry Pervasives.stdout) entries
+  );
+  entries
diff --git a/builder/simplestreams_parser.mli b/builder/simplestreams_parser.mli
new file mode 100644
index 0000000..a4b91ba
--- /dev/null
+++ b/builder/simplestreams_parser.mli
@@ -0,0 +1,19 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t ->
Sources.source -> Index.index
diff --git a/builder/sources.ml b/builder/sources.ml
index b21e8fc..149db6f 100644
--- a/builder/sources.ml
+++ b/builder/sources.ml
@@ -31,6 +31,7 @@ type source = {
 }
 and source_format  | FormatNative
+| FormatSimpleStreams

 module StringSet = Set.Make (String)

@@ -82,6 +83,14 @@ let parse_conf file            try
             (match (List.assoc ("format", None) fields) with
             | "native" | "" -> FormatNative
+            | "simplestreams" as fmt ->
+              if not (Yajl.yajl_is_available ()) then (
+                if verbose () then (
+                  eprintf (f_"%s: repository type '%s' not
supported (missing YAJL support), skipping it\n") prog fmt;
+                );
+                invalid_arg fmt
+              ) else
+                FormatSimpleStreams
             | fmt ->
               if verbose () then (
                 eprintf (f_"%s: unknown repository type '%s' in
%s, skipping it\n") prog fmt file;
diff --git a/builder/sources.mli b/builder/sources.mli
index e861310..e621a9f 100644
--- a/builder/sources.mli
+++ b/builder/sources.mli
@@ -25,5 +25,6 @@ type source = {
 }
 and source_format  | FormatNative
+| FormatSimpleStreams

 val read_sources : unit -> source list
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index fc49d4d..0de643a 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1181,6 +1181,13 @@ The possible values are:
 The native format of the C<virt-builder> repository.  See also
 L</Creating and signing the index file> below.

+=item B<simplestreams>
+
+The URI represents the root of a Simple Streams v1.0 tree of metadata.
+
+For more information about Simple Streams, see also
+L<https://launchpad.net/simplestreams>.
+
 =back

 If not present, the assumed value is C<native>.
diff --git a/builder/yajl-c.c b/builder/yajl-c.c
new file mode 100644
index 0000000..cb47efa
--- /dev/null
+++ b/builder/yajl-c.c
@@ -0,0 +1,141 @@
+/* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA.
+ */
+
+#include <config.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#if HAVE_YAJL
+#include <yajl/yajl_tree.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+
+#define Val_none (Val_int (0))
+
+extern value virt_builder_yajl_tree_parse (value stringv);
+extern value virt_builder_yajl_is_available (value unit);
+
+#if HAVE_YAJL
+static value
+convert_yajl_value (yajl_val val, int level)
+{
+  CAMLparam0 ();
+  CAMLlocal4 (rv, lv, v, sv);
+
+  if (level > 20)
+    caml_invalid_argument ("too many levels of object/array
nesting");
+
+  if (YAJL_IS_OBJECT (val)) {
+    size_t len = YAJL_GET_OBJECT(val)->len;
+    size_t i;
+    rv = caml_alloc (1, 3);
+    lv = caml_alloc_tuple (len);
+    for (i = 0; i < len; ++i) {
+      v = caml_alloc_tuple (2);
+      sv = caml_copy_string (YAJL_GET_OBJECT(val)->keys[i]);
+      Store_field (v, 0, sv);
+      sv = convert_yajl_value (YAJL_GET_OBJECT(val)->values[i], level + 1);
+      Store_field (v, 1, sv);
+      Store_field (lv, i, v);
+    }
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_ARRAY (val)) {
+    size_t len = YAJL_GET_ARRAY(val)->len;
+    size_t i;
+    rv = caml_alloc (1, 4);
+    lv = caml_alloc_tuple (len);
+    for (i = 0; i < len; ++i) {
+      v = convert_yajl_value (YAJL_GET_ARRAY(val)->values[i], level + 1);
+      Store_field (lv, i, v);
+    }
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_STRING (val)) {
+    rv = caml_alloc (1, 0);
+    v = caml_copy_string (YAJL_GET_STRING(val));
+    Store_field (rv, 0, v);
+  } else if (YAJL_IS_DOUBLE (val)) {
+    rv = caml_alloc (1, 2);
+    lv = caml_alloc_tuple (1);
+    Store_double_field (lv, 0, YAJL_GET_DOUBLE(val));
+    Store_field (rv, 0, lv);
+  } else if (YAJL_IS_INTEGER (val)) {
+    rv = caml_alloc (1, 1);
+    v = caml_copy_int64 (YAJL_GET_INTEGER(val));
+    Store_field (rv, 0, v);
+  } else if (YAJL_IS_TRUE (val)) {
+    rv = caml_alloc (1, 5);
+    Store_field (rv, 0, Val_true);
+  } else if (YAJL_IS_FALSE (val)) {
+    rv = caml_alloc (1, 5);
+    Store_field (rv, 0, Val_false);
+  } else
+    rv = Val_none;
+
+  CAMLreturn (rv);
+}
+
+value
+virt_builder_yajl_is_available (value unit)
+{
+  return Val_true;
+}
+
+value
+virt_builder_yajl_tree_parse (value stringv)
+{
+  CAMLparam1 (stringv);
+  CAMLlocal1 (rv);
+  yajl_val tree;
+  char error_buf[256];
+
+  tree = yajl_tree_parse (String_val (stringv), error_buf, sizeof error_buf);
+  if (tree == NULL) {
+    char buf[256 + sizeof error_buf];
+    if (strlen (error_buf) > 0)
+      snprintf (buf, sizeof buf, "JSON parse error: %s", error_buf);
+    else
+      snprintf (buf, sizeof buf, "unknown JSON parse error");
+    caml_invalid_argument (buf);
+  }
+
+  rv = convert_yajl_value (tree, 1);
+  yajl_tree_free (tree);
+
+  CAMLreturn (rv);
+}
+
+#else
+
+value
+virt_builder_yajl_is_available (value unit)
+{
+  return Val_false;
+}
+
+value
+virt_builder_yajl_tree_parse (value stringv)
+{
+  caml_invalid_argument ("virt-builder was compiled without yajl
support");
+}
+
+#endif
diff --git a/builder/yajl.ml b/builder/yajl.ml
new file mode 100644
index 0000000..f2d5c2b
--- /dev/null
+++ b/builder/yajl.ml
@@ -0,0 +1,30 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type yajl_val +| Yajl_null
+| Yajl_string of string
+| Yajl_number of int64
+| Yajl_double of float
+| Yajl_object of (string * yajl_val) array
+| Yajl_array of yajl_val array
+| Yajl_bool of bool
+
+external yajl_is_available : unit -> bool =
"virt_builder_yajl_is_available" "noalloc"
+
+external yajl_tree_parse : string -> yajl_val =
"virt_builder_yajl_tree_parse"
diff --git a/builder/yajl.mli b/builder/yajl.mli
new file mode 100644
index 0000000..aaa9389
--- /dev/null
+++ b/builder/yajl.mli
@@ -0,0 +1,33 @@
+(* virt-builder
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type yajl_val +| Yajl_null
+| Yajl_string of string
+| Yajl_number of int64
+| Yajl_double of float
+| Yajl_object of (string * yajl_val) array
+| Yajl_array of yajl_val array
+| Yajl_bool of bool
+
+val yajl_is_available : unit -> bool
+(** Is YAJL built in? If not, calling any of the other yajl_*
+    functions will result in an error. *)
+
+val yajl_tree_parse : string -> yajl_val
+(** Parse the JSON string. *)
diff --git a/po/POTFILES b/po/POTFILES
index 7f1580c..8fb68a4 100644
--- a/po/POTFILES
+++ b/po/POTFILES
@@ -6,6 +6,7 @@ builder/index-struct.c
 builder/index-validate.c
 builder/pxzcat-c.c
 builder/setlocale-c.c
+builder/yajl-c.c
 cat/cat.c
 cat/filesystems.c
 cat/log.c
diff --git a/po/POTFILES-ml b/po/POTFILES-ml
index 7933c8e..87f10ee 100644
--- a/po/POTFILES-ml
+++ b/po/POTFILES-ml
@@ -12,8 +12,10 @@ builder/paths.ml
 builder/pxzcat.ml
 builder/setlocale.ml
 builder/sigchecker.ml
+builder/simplestreams_parser.ml
 builder/sources.ml
 builder/utils.ml
+builder/yajl.ml
 customize/crypt.ml
 customize/customize_cmdline.ml
 customize/customize_main.ml
--
2.1.0