Libguestfs - Aug 2016 - [PATCH] tests: use /etc/os-release in test-harder

Hi,

let's make supermin use /etc/os-release as primary source instead of
the various release files in /etc; apparently distros (e.g. openSUSE)
are starting removing them.

Thanks,


Pino Toscano (2):
  Add simple handling of /etc/os-release
  Use os-release to detect the distro

 src/Makefile.am    |  3 +++
 src/dpkg.ml        |  3 ++-
 src/os_release.ml  | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/os_release.mli | 26 ++++++++++++++++++
 src/pacman.ml      |  5 ++--
 src/rpm.ml         | 15 ++++++-----
 6 files changed, 121 insertions(+), 9 deletions(-)
 create mode 100644 src/os_release.ml
 create mode 100644 src/os_release.mli

-- 
2.7.4

Introduce a simple module to read and cache fields of /etc/os-release
that might be needed, and there is only ID for now.
---
 src/Makefile.am    |  3 +++
 src/os_release.ml  | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/os_release.mli | 26 ++++++++++++++++++
 3 files changed, 107 insertions(+)
 create mode 100644 src/os_release.ml
 create mode 100644 src/os_release.mli

diff --git a/src/Makefile.am b/src/Makefile.am
index 11adf31..767117f 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -47,6 +47,8 @@ SOURCES = \
 	utils.ml \
 	utils.mli \
 	types.ml \
+	os_release.ml \
+	os_release.mli \
 	package_handler.ml \
 	package_handler.mli \
 	rpm.ml \
@@ -71,6 +73,7 @@ SOURCES_ML = \
 	config.ml \
 	utils.ml \
 	types.ml \
+	os_release.ml \
 	package_handler.ml \
 	rpm.ml \
 	dpkg.ml \
diff --git a/src/os_release.ml b/src/os_release.ml
new file mode 100644
index 0000000..b2de259
--- /dev/null
+++ b/src/os_release.ml
@@ -0,0 +1,78 @@
+(* supermin 5
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *)
+
+open Utils
+
+let split sep str +  let len = String.length sep in
+  let seplen = String.length str in
+  let i = find str sep in
+  if i = -1 then str, ""
+  else (
+    String.sub str 0 i, String.sub str (i + len) (seplen - i - len)
+  )
+
+type os_release = {
+  id : string;
+}
+
+let data = ref None
+let parsed = ref false
+
+let rec get_data () +  if !parsed = false then (
+    data := parse ();
+    parsed := true;
+  );
+
+  !data
+
+and parse () +  let file = "/etc/os-release" in
+  if Sys.file_exists file then (
+    let chan = open_in file in
+    let lines = input_all_lines chan in
+    close_in chan;
+    let lines = List.filter ((<>) "") lines in
+    let lines = List.filter (fun s -> s.[0] <> '#') lines in
+
+    let id = ref "" in
+
+    List.iter (
+      fun line ->
+        let field, value = split "=" line in
+        let value +          let len = String.length value in
+          if len > 1 &&
+             ((value.[0] = '"' && value.[len-1] =
'"') ||
+              (value.[0] = '\'' && value.[len-1] =
'\'')) then
+            String.sub value 1 (len - 2)
+          else value in
+        match field with
+        | "ID" -> id := value
+        | _ -> ()
+    ) lines;
+
+    Some { id = !id; }
+  ) else
+    None
+
+let get_id () +  match get_data () with
+  | None -> ""
+  | Some d -> d.id
diff --git a/src/os_release.mli b/src/os_release.mli
new file mode 100644
index 0000000..2ae349b
--- /dev/null
+++ b/src/os_release.mli
@@ -0,0 +1,26 @@
+(* supermin 5
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *)
+
+(** Handling of /etc/os-release. *)
+
+val get_id : unit -> string
+(** Get the value of the "ID" field from the /etc/os-release file
+    on the current system.
+
+    An empty string is returned if the file does not exist or cannot
+    be read. *)
-- 
2.7.4

Check the ID field in /etc/os-release on the current system, before
checking for the other old-style release-/version-like files in /etc.
Some distributions (openSUSE Thumbleweed) are starting to remove them,
breaking the supermin detection.
---
 src/dpkg.ml   |  3 ++-
 src/pacman.ml |  5 +++--
 src/rpm.ml    | 15 +++++++++------
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/dpkg.ml b/src/dpkg.ml
index 70acfa2..1e785de 100644
--- a/src/dpkg.ml
+++ b/src/dpkg.ml
@@ -28,7 +28,8 @@ let dpkg_detect ()      Config.dpkg_query <>
"no" &&
     Config.dpkg_divert <> "no" &&
     Config.apt_get <> "no" &&
-    try (stat "/etc/debian_version").st_kind = S_REG with Unix_error
_ -> false
+    (List.mem (Os_release.get_id ()) [ "debian"; "ubuntu" ]
||
+     try (stat "/etc/debian_version").st_kind = S_REG with Unix_error
_ -> false)
 
 let dpkg_primary_arch = ref ""
 let settings = ref no_settings
diff --git a/src/pacman.ml b/src/pacman.ml
index 3340fa6..c35668a 100644
--- a/src/pacman.ml
+++ b/src/pacman.ml
@@ -24,8 +24,9 @@ open Package_handler
 
 let pacman_detect ()    Config.pacman <> "no" &&
Config.fakeroot <> "no" &&
-    (stat "/etc/arch-release").st_kind = S_REG &&
-    Config.pacman_g2 = "no" (* not Frugalware with pacman-g2 *)
+    (Os_release.get_id () = "arch" ||
+     ((stat "/etc/arch-release").st_kind = S_REG &&
+      Config.pacman_g2 = "no")) (* not Frugalware with pacman-g2 *)
 
 let settings = ref no_settings
 
diff --git a/src/rpm.ml b/src/rpm.ml
index a5dc67a..e409e37 100644
--- a/src/rpm.ml
+++ b/src/rpm.ml
@@ -31,21 +31,24 @@ let stringset_of_list pkgs  let fedora_detect ()   
Config.rpm <> "no" && Config.rpm2cpio <>
"no" && rpm_is_available () &&
     (Config.yumdownloader <> "no" || Config.dnf <>
"no") &&
-    try
-      (stat "/etc/redhat-release").st_kind = S_REG ||
-      (stat "/etc/fedora-release").st_kind = S_REG
-    with Unix_error _ -> false
+    (List.mem (Os_release.get_id ()) [ "fedora"; "rhel";
"centos" ] ||
+     try
+       (stat "/etc/redhat-release").st_kind = S_REG ||
+       (stat "/etc/fedora-release").st_kind = S_REG
+     with Unix_error _ -> false)
 
 let opensuse_detect ()    Config.rpm <> "no" &&
Config.rpm2cpio <> "no" && rpm_is_available ()
&&
     Config.zypper <> "no" &&
-    try (stat "/etc/SuSE-release").st_kind = S_REG with Unix_error _
-> false
+    (List.mem (Os_release.get_id ()) [ "opensuse"; "sled";
"sles" ] ||
+     try (stat "/etc/SuSE-release").st_kind = S_REG with Unix_error _
-> false)
 
 let mageia_detect ()    Config.rpm <> "no" &&
Config.rpm2cpio <> "no" && rpm_is_available ()
&&
     Config.urpmi <> "no" &&
     Config.fakeroot <> "no" &&
-    try (stat "/etc/mageia-release").st_kind = S_REG with Unix_error
_ -> false
+    (Os_release.get_id () = "mageia" ||
+     try (stat "/etc/mageia-release").st_kind = S_REG with Unix_error
_ -> false)
 
 let ibm_powerkvm_detect ()    Config.rpm <> "no" &&
Config.rpm2cpio <> "no" && rpm_is_available ()
&&
-- 
2.7.4

On Wed, 2016-08-31 at 15:05 +0200, Pino Toscano wrote:
> Check the ID field in /etc/os-release on the current system, before
> checking for the other old-style release-/version-like files in /etc.
> Some distributions (openSUSE Thumbleweed) are starting to remove them,
> breaking the supermin detection.
> ---
>  src/dpkg.ml   |  3 ++-
>  src/pacman.ml |  5 +++--
>  src/rpm.ml    | 15 +++++++++------
>  3 files changed, 14 insertions(+), 9 deletions(-)
> 
> diff --git a/src/dpkg.ml b/src/dpkg.ml
> index 70acfa2..1e785de 100644
> --- a/src/dpkg.ml
> +++ b/src/dpkg.ml
> @@ -28,7 +28,8 @@ let dpkg_detect () >      Config.dpkg_query <>
"no" &&
>      Config.dpkg_divert <> "no" &&
>      Config.apt_get <> "no" &&
> -    try (stat "/etc/debian_version").st_kind = S_REG with
Unix_error _ -> false
> +    (List.mem (Os_release.get_id ()) [ "debian";
"ubuntu" ] ||
> +     try (stat "/etc/debian_version").st_kind = S_REG with
Unix_error _ -> false)
>  
>  let dpkg_primary_arch = ref ""
>  let settings = ref no_settings
> diff --git a/src/pacman.ml b/src/pacman.ml
> index 3340fa6..c35668a 100644
> --- a/src/pacman.ml
> +++ b/src/pacman.ml
> @@ -24,8 +24,9 @@ open Package_handler
>  
>  let pacman_detect () >    Config.pacman <> "no"
&& Config.fakeroot <> "no" &&
> -    (stat "/etc/arch-release").st_kind = S_REG &&
> -    Config.pacman_g2 = "no" (* not Frugalware with pacman-g2 *)
> +    (Os_release.get_id () = "arch" ||
> +     ((stat "/etc/arch-release").st_kind = S_REG &&
> +      Config.pacman_g2 = "no")) (* not Frugalware with pacman-g2
*)
>  
>  let settings = ref no_settings
>  
> diff --git a/src/rpm.ml b/src/rpm.ml
> index a5dc67a..e409e37 100644
> --- a/src/rpm.ml
> +++ b/src/rpm.ml
> @@ -31,21 +31,24 @@ let stringset_of_list pkgs >  let fedora_detect ()
>    Config.rpm <> "no" && Config.rpm2cpio <>
"no" && rpm_is_available () &&
>      (Config.yumdownloader <> "no" || Config.dnf <>
"no") &&
> -    try
> -      (stat "/etc/redhat-release").st_kind = S_REG ||
> -      (stat "/etc/fedora-release").st_kind = S_REG
> -    with Unix_error _ -> false
> +    (List.mem (Os_release.get_id ()) [ "fedora";
"rhel"; "centos" ] ||
> +     try
> +       (stat "/etc/redhat-release").st_kind = S_REG ||
> +       (stat "/etc/fedora-release").st_kind = S_REG
> +     with Unix_error _ -> false)
>  
>  let opensuse_detect () >    Config.rpm <> "no"
&& Config.rpm2cpio <> "no" && rpm_is_available
() &&
>      Config.zypper <> "no" &&
> -    try (stat "/etc/SuSE-release").st_kind = S_REG with
Unix_error _ -> false
> +    (List.mem (Os_release.get_id ()) [ "opensuse";
"sled"; "sles" ] ||
> +     try (stat "/etc/SuSE-release").st_kind = S_REG with
Unix_error _ -> false)
>  
>  let mageia_detect () >    Config.rpm <> "no" &&
Config.rpm2cpio <> "no" && rpm_is_available ()
&&
>      Config.urpmi <> "no" &&
>      Config.fakeroot <> "no" &&
> -    try (stat "/etc/mageia-release").st_kind = S_REG with
Unix_error _ -> false
> +    (Os_release.get_id () = "mageia" ||
> +     try (stat "/etc/mageia-release").st_kind = S_REG with
Unix_error _ -> false)
>  
>  let ibm_powerkvm_detect () >    Config.rpm <> "no"
&& Config.rpm2cpio <> "no" && rpm_is_available
() &&
Looks good to me, at least for the openSUSE / SLE parts.

--
Cedric

Check the ID in /etc/os-release before checking the other release files,
so it's possible to handle distros without them.

Also, make sure it is skipped if the value read from os-release is not
handled when getting the list of packages.
---
 tests/test-harder.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/test-harder.sh b/tests/test-harder.sh
index ea5dfc8..7e1b8df 100755
--- a/tests/test-harder.sh
+++ b/tests/test-harder.sh
@@ -25,7 +25,14 @@ set -e
 # NOTE:  This test will only work if the $pkgs listed below
 # for your distro are installed on the host.  SEE LIST BELOW.
 
-if [ -f /etc/arch-release ]; then
+if [ -f /etc/os-release ]; then
+    distro=$(. /etc/os-release && echo $ID)
+    case "$distro" in
+        fedora|rhel|centos) distro=redhat ;;
+        opensuse|sled|sles) distro=suse ;;
+        ubuntu) distro=debian ;;
+    esac
+elif [ -f /etc/arch-release ]; then
     distro=arch
 elif [ -f /etc/debian_version ]; then
     distro=debian
@@ -63,6 +70,10 @@ case $distro in
     ibm-powerkvm)
 	pkgs="augeas hivex tar"
 	;;
+    *)
+	echo "Unhandled distro '$distro'"
+	exit 77
+	;;
 esac
 
 test "$USE_NETWORK" = 1 || USE_INSTALLED=--use-installed
-- 
2.7.4

Greetings,

I built a small proof-of-concept and I've been suggested to share it 
with the community.

The tool consists of a vulnerability scanner based on Libguestfs.

The tool lists all the installed applications within a disk image and 
queries a CVE database via REST interface. The data gets aggregated in 
order to provide a report of the vulnerable applications within the disk 
image.

Here's a concrete example:
http://pastebin.com/w6DZkwCg

A possible use case could be the vulnerability assessment and management 
of Cloud instances.

The tool is part of a library I've been building to help automating 
security assessment and forensics analysis of disk images.
https://github.com/noxdafox/vminspect

I did not test it much yet. Therefore, it might raise several false 
positives or miss important vulnerabilities but considering it's ~ 100 
lines of Python code, I'd say is a good starting point.

The tool is relying on cve-search REST APIs to retrieve the 
vulnerability list.
https://github.com/adulau/cve-search

On Wed, 2016-08-31 at 15:05 +0200, Pino Toscano wrote:
> Hi,
> 
> let's make supermin use /etc/os-release as primary source instead of
> the various release files in /etc; apparently distros (e.g. openSUSE)
> are starting removing them.
> 
> Thanks,
> 
> 
> Pino Toscano (2):
>   Add simple handling of /etc/os-release
>   Use os-release to detect the distro
> 
>  src/Makefile.am    |  3 +++
>  src/dpkg.ml        |  3 ++-
>  src/os_release.ml  | 78
++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  src/os_release.mli | 26 ++++++++++++++++++
>  src/pacman.ml      |  5 ++--
>  src/rpm.ml         | 15 ++++++-----
>  6 files changed, 121 insertions(+), 9 deletions(-)
>  create mode 100644 src/os_release.ml
>  create mode 100644 src/os_release.mli
> 
ACK to the whole series. Just tested it and backported it to Factory packages.
openSUSE updated supermin package will arrive soon.

--
Cedric

On Wednesday, 31 August 2016 15:05:34 CEST Pino Toscano
wrote:
> let's make supermin use /etc/os-release as primary source instead of
> the various release files in /etc; apparently distros (e.g. openSUSE)
> are starting removing them.
Ping.

Thanks,
-- 
Pino Toscano

On Wed, Aug 31, 2016 at 03:05:34PM +0200, Pino Toscano
wrote:
> Hi,
> 
> let's make supermin use /etc/os-release as primary source instead of
> the various release files in /etc; apparently distros (e.g. openSUSE)
> are starting removing them.
> 
> Thanks,
> 
> 
> Pino Toscano (2):
>   Add simple handling of /etc/os-release
>   Use os-release to detect the distro
> 
>  src/Makefile.am    |  3 +++
>  src/dpkg.ml        |  3 ++-
>  src/os_release.ml  | 78
++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  src/os_release.mli | 26 ++++++++++++++++++
>  src/pacman.ml      |  5 ++--
>  src/rpm.ml         | 15 ++++++-----
>  6 files changed, 121 insertions(+), 9 deletions(-)
>  create mode 100644 src/os_release.ml
>  create mode 100644 src/os_release.mli
Yes, this looks good.  ACK series.

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
libguestfs lets you edit virtual machines.  Supports shell scripting,
bindings from many languages.  http://libguestfs.org