Richard W.M. Jones
2017-Mar-23 14:31 UTC
[Libguestfs] [PATCH] p2v: Use lscpu instead of libvirt to get CPU information.
Don't get the CPU information from libvirt, because including libvirt and all dependencies in the virt-p2v ISO bloats everything. Instead get most of the information we need from the util-linux program 'lscpu'. Unfortunately the CPU model cannot be retrieved. Example output: $ ./run virt-p2v --cmdline="p2v.dump_config_and_exit" [...] cpu vendor . . . Intel cpu sockets . . 2 cpu cores . . . 8 cpu threads . . 1 flags . . . . . acpi apic pae This updates commit 963d6c3be7cd91c0373f67cfdd95c4f1dad1452f. --- p2v/Makefile.am | 11 +- p2v/cpuid.c | 331 ++++++++++++++++++++-------------------------------- p2v/dependencies.m4 | 5 - 3 files changed, 128 insertions(+), 219 deletions(-) diff --git a/p2v/Makefile.am b/p2v/Makefile.am index 726916027..94c649a8e 100644 --- a/p2v/Makefile.am +++ b/p2v/Makefile.am @@ -100,7 +100,6 @@ virt_p2v_CPPFLAGS = \ virt_p2v_CFLAGS = \ -pthread \ $(WARN_CFLAGS) $(WERROR_CFLAGS) \ - $(LIBVIRT_CFLAGS) \ $(PCRE_CFLAGS) \ $(LIBXML2_CFLAGS) \ $(GTK_CFLAGS) \ @@ -109,7 +108,6 @@ virt_p2v_CFLAGS = \ virt_p2v_LDADD = \ $(top_builddir)/common/utils/libutils.la \ $(top_builddir)/common/miniexpect/libminiexpect.la \ - $(LIBVIRT_LIBS) \ $(PCRE_LIBS) \ $(LIBXML2_LIBS) \ $(GTK_LIBS) \ @@ -126,16 +124,9 @@ dependencies_files = \ dependencies.redhat \ dependencies.suse -if HAVE_LIBVIRT -dependencies_have_libvirt = -DHAVE_LIBVIRT=1 -endif - $(dependencies_files): dependencies.m4 define=`echo $@ | $(SED) 's/dependencies.//;y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`; \ - m4 -D$$define=1 \ - -DGTK_VERSION=$(GTK_VERSION) \ - $(dependencies_have_libvirt) \ - $< > $@-t + m4 -D$$define=1 -DGTK_VERSION=$(GTK_VERSION) $< > $@-t mv $@-t $@ # Support files needed by the virt-p2v-make-* scripts. diff --git a/p2v/cpuid.c b/p2v/cpuid.c index 13b61b050..12720552d 100644 --- a/p2v/cpuid.c +++ b/p2v/cpuid.c @@ -17,20 +17,17 @@ */ /** - * Process CPU capabilities into libvirt-compatible C<E<lt>cpuE<gt>> data. + * Find CPU vendor, topology and some CPU flags. * - * If libvirt is available at compile time then this is quite - * simple - libvirt API C<virConnectGetCapabilities> provides - * a C<E<lt>hostE<ge>> element which has mostly what we need. + * lscpu (from util-linux) provides CPU vendor, topology and flags. * - * Flags C<acpi>, C<apic>, C<pae> still have to be parsed out of - * F</proc/cpuinfo> because these will not necessarily be present in - * the libvirt capabilities directly (they are implied by the - * processor model, requiring a complex lookup in the CPU map). + * ACPI can be read by seeing if F</sys/firmware/acpi> exists. * - * Note that #vCPUs and amount of RAM is handled by F<main.c>. + * CPU model is essentially impossible to get without using libvirt, + * but we cannot use libvirt for the reasons outlined in this message: + * https://www.redhat.com/archives/libvirt-users/2017-March/msg00071.html * - * See: L<https://libvirt.org/formatdomain.html#elementsCPU> + * Note that #vCPUs and amount of RAM is handled by F<main.c>. */ #include <config.h> @@ -40,15 +37,10 @@ #include <stdarg.h> #include <string.h> #include <errno.h> +#include <error.h> #include <libintl.h> -#ifdef HAVE_LIBVIRT -#include <libvirt/libvirt.h> -#include <libvirt/virterror.h> -#endif - -#include <libxml/xpath.h> - +#include "c-ctype.h" #include "getprogname.h" #include "ignore-value.h" @@ -65,235 +57,166 @@ free_cpu_config (struct cpu_config *cpu) } /** - * Read flags from F</proc/cpuinfo>. + * Get the output of lscpu as a list of (key, value) pairs (as a + * flattened list of strings). */ -static void -cpuinfo_flags (struct cpu_config *cpu) +static char ** +get_lscpu (void) { const char *cmd; CLEANUP_PCLOSE FILE *fp = NULL; - CLEANUP_FREE char *flag = NULL; + CLEANUP_FREE char *line = NULL; ssize_t len; size_t buflen = 0; + char **ret = NULL; + size_t ret_size = 0; - /* Get the flags, one per line. */ - cmd = "< /proc/cpuinfo " -#if defined(__arm__) - "grep ^Features" -#else - "grep ^flags" -#endif - " | awk '{ for (i = 3; i <= NF; ++i) { print $i }; exit }'"; + cmd = "lscpu"; fp = popen (cmd, "re"); if (fp == NULL) { - perror ("/proc/cpuinfo"); - return; + perror (cmd); + return NULL; } - while (errno = 0, (len = getline (&flag, &buflen, fp)) != -1) { - if (len > 0 && flag[len-1] == '\n') - flag[len-1] = '\0'; - - if (STREQ (flag, "acpi")) - cpu->acpi = 1; - else if (STREQ (flag, "apic")) - cpu->apic = 1; - else if (STREQ (flag, "pae")) - cpu->pae = 1; + ret = malloc (sizeof (char *)); + if (ret == NULL) error (EXIT_FAILURE, errno, "malloc"); + ret[0] = NULL; + + while (errno = 0, (len = getline (&line, &buflen, fp)) != -1) { + char *p; + char *key, *value; + + if (len > 0 && line[len-1] == '\n') + line[len-1] = '\0'; + + /* Split the line at the first ':' character. */ + p = strchr (line, ':'); + if (p == NULL) + continue; + + *p = '\0'; + key = strdup (line); + /* Skip leading whitespace in the value. */ + for (++p; *p && c_isspace (*p); ++p) + ; + value = strdup (p); + + /* Add key and value to the list, and trailing NULL pointer. */ + ret_size += 2; + ret = realloc (ret, (ret_size + 1) * sizeof (char *)); + if (ret == NULL) error (EXIT_FAILURE, errno, "realloc"); + ret[ret_size-2] = key; + ret[ret_size-1] = value; + ret[ret_size] = NULL; } if (errno) { - perror ("getline"); - return; + perror (cmd); + guestfs_int_free_string_list (ret); + return NULL; } + + return ret; } -#ifdef HAVE_LIBVIRT +/** + * Read a single field from lscpu output. + * + * If the field does not exist, returns C<NULL>. + */ +static const char * +get_field (char **lscpu, const char *key) +{ + size_t i; + + for (i = 0; lscpu[i] != NULL; i += 2) { + if (STREQ (lscpu[i], key)) + return lscpu[i+1]; + } + + return NULL; +} +/** + * Read the CPU vendor from lscpu output. + */ static void -ignore_errors (void *ignore, virErrorPtr ignore2) +get_vendor (char **lscpu, struct cpu_config *cpu) { - /* empty */ + const char *vendor = get_field (lscpu, "Vendor ID"); + + if (vendor) { + /* Note this mapping comes from /usr/share/libvirt/cpu_map.xml */ + if (STREQ (vendor, "GenuineIntel")) + cpu->vendor = strdup ("Intel"); + else if (STREQ (vendor, "AuthenticAMD")) + cpu->vendor = strdup ("AMD"); + /* Currently aarch64 lscpu has no Vendor ID XXX. */ + } } -static void libvirt_error (const char *fs, ...) __attribute__((format (printf,1,2))); - +/** + * Read the CPU topology from lscpu output. + */ static void -libvirt_error (const char *fs, ...) +get_topology (char **lscpu, struct cpu_config *cpu) { - va_list args; - CLEANUP_FREE char *msg = NULL; - int len; - virErrorPtr err; - - va_start (args, fs); - len = vasprintf (&msg, fs, args); - va_end (args); - - if (len < 0) goto fallback; - - /* In all recent libvirt, this retrieves the thread-local error. */ - err = virGetLastError (); - if (err) - fprintf (stderr, - "%s: %s: %s [code=%d int1=%d]\n", - getprogname (), msg, err->message, err->code, err->int1); - else - fallback: - fprintf (stderr, "%s: %s\n", getprogname (), msg); + const char *v; + + v = get_field (lscpu, "Socket(s)"); + if (v) + ignore_value (sscanf (v, "%u", &cpu->sockets)); + v = get_field (lscpu, "Core(s) per socket"); + if (v) + ignore_value (sscanf (v, "%u", &cpu->cores)); + v = get_field (lscpu, "Thread(s) per core"); + if (v) + ignore_value (sscanf (v, "%u", &cpu->threads)); } /** - * Read the capabilities from libvirt and parse out the fields - * we care about. + * Read some important flags from lscpu output. */ static void -libvirt_capabilities (struct cpu_config *cpu) +get_flags (char **lscpu, struct cpu_config *cpu) { - virConnectPtr conn; - CLEANUP_FREE char *capabilities_xml = NULL; - CLEANUP_XMLFREEDOC xmlDocPtr doc = NULL; - CLEANUP_XMLXPATHFREECONTEXT xmlXPathContextPtr xpathCtx = NULL; - CLEANUP_XMLXPATHFREEOBJECT xmlXPathObjectPtr xpathObj = NULL; - const char *xpathexpr; - xmlNodeSetPtr nodes; - size_t nr_nodes, i; - xmlNodePtr node; - - /* Connect to libvirt and get the capabilities XML. */ - conn = virConnectOpenReadOnly (NULL); - if (!conn) { - libvirt_error (_("could not connect to libvirt")); - return; - } + const char *flags; - /* Suppress default behaviour of printing errors to stderr. Note - * you can't set this to NULL to ignore errors; setting it to NULL - * restores the default error handler ... - */ - virConnSetErrorFunc (conn, NULL, ignore_errors); - - capabilities_xml = virConnectGetCapabilities (conn); - if (!capabilities_xml) { - libvirt_error (_("could not get libvirt capabilities")); - virConnectClose (conn); - return; - } - - /* Parse the capabilities XML with libxml2. */ - doc = xmlReadMemory (capabilities_xml, strlen (capabilities_xml), - NULL, NULL, XML_PARSE_NONET); - if (doc == NULL) { - fprintf (stderr, - _("%s: unable to parse capabilities XML returned by libvirt\n"), - getprogname ()); - virConnectClose (conn); - return; - } - - xpathCtx = xmlXPathNewContext (doc); - if (xpathCtx == NULL) { - fprintf (stderr, _("%s: unable to create new XPath context\n"), - getprogname ()); - virConnectClose (conn); - return; - } - - /* Get the CPU vendor. */ - xpathexpr = "/capabilities/host/cpu/vendor/text()"; - xpathObj = xmlXPathEvalExpression (BAD_CAST xpathexpr, xpathCtx); - if (xpathObj == NULL) { - fprintf (stderr, _("%s: unable to evaluate xpath expression: %s\n"), - getprogname (), xpathexpr); - virConnectClose (conn); - return; - } - nodes = xpathObj->nodesetval; - nr_nodes = nodes->nodeNr; - if (nr_nodes > 0) { - node = nodes->nodeTab[0]; - cpu->vendor = (char *) xmlNodeGetContent (node); - } - - /* Get the CPU model. */ - xmlXPathFreeObject (xpathObj); - xpathexpr = "/capabilities/host/cpu/model/text()"; - xpathObj = xmlXPathEvalExpression (BAD_CAST xpathexpr, xpathCtx); - if (xpathObj == NULL) { - fprintf (stderr, _("%s: unable to evaluate xpath expression: %s\n"), - getprogname (), xpathexpr); - virConnectClose (conn); - return; - } - nodes = xpathObj->nodesetval; - nr_nodes = nodes->nodeNr; - if (nr_nodes > 0) { - node = nodes->nodeTab[0]; - cpu->model = (char *) xmlNodeGetContent (node); - } + flags = get_field (lscpu, "Flags"); + if (flags) { + cpu->apic = strstr (flags, " apic ") != NULL; + cpu->pae = strstr (flags, " pae ") != NULL; - /* Get the topology. Note the XPath expression returns all - * attributes of the <topology> node. - */ - xmlXPathFreeObject (xpathObj); - xpathexpr = "/capabilities/host/cpu/topology/@*"; - xpathObj = xmlXPathEvalExpression (BAD_CAST xpathexpr, xpathCtx); - if (xpathObj == NULL) { - fprintf (stderr, _("%s: unable to evaluate xpath expression: %s\n"), - getprogname (), xpathexpr); - virConnectClose (conn); - return; + /* aarch64 /proc/cpuinfo has a "Features" field, but lscpu does + * not expose it. However aarch64 Features does not contain any + * of the interesting flags above. + */ } - nodes = xpathObj->nodesetval; - nr_nodes = nodes->nodeNr; - /* Iterate over the attributes of the <topology> node. */ - for (i = 0; i < nr_nodes; ++i) { - node = nodes->nodeTab[i]; - - if (node->type == XML_ATTRIBUTE_NODE) { - xmlAttrPtr attr = (xmlAttrPtr) node; - CLEANUP_FREE char *content = NULL; - unsigned *up; - - if (STREQ ((const char *) attr->name, "sockets")) { - up = &cpu->sockets; - parse_attr: - *up = 0; - content = (char *) xmlNodeListGetString (doc, attr->children, 1); - if (content) - ignore_value (sscanf (content, "%u", up)); - } - else if (STREQ ((const char *) attr->name, "cores")) { - up = &cpu->cores; - goto parse_attr; - } - else if (STREQ ((const char *) attr->name, "threads")) { - up = &cpu->threads; - goto parse_attr; - } - } - } - - virConnectClose (conn); } -#else /* !HAVE_LIBVIRT */ - +/** + * Find out if the system uses ACPI. + */ static void -libvirt_capabilities (struct cpu_config *cpu) +get_acpi (struct cpu_config *cpu) { - fprintf (stderr, - _("%s: program was compiled without libvirt support\n"), - getprogname ()); + cpu->acpi = access ("/sys/firmware/acpi", F_OK) == 0; } -#endif /* !HAVE_LIBVIRT */ - void get_cpu_config (struct cpu_config *cpu) { + CLEANUP_FREE_STRING_LIST char **lscpu = NULL; + free_cpu_config (cpu); - libvirt_capabilities (cpu); - cpuinfo_flags (cpu); + + lscpu = get_lscpu (); + if (lscpu != NULL) { + get_vendor (lscpu, cpu); + get_topology (lscpu, cpu); + get_flags (lscpu, cpu); + } + + get_acpi (cpu); } diff --git a/p2v/dependencies.m4 b/p2v/dependencies.m4 index 0db2f85d4..02ca87c19 100644 --- a/p2v/dependencies.m4 +++ b/p2v/dependencies.m4 @@ -25,8 +25,6 @@ ifelse(REDHAT,1, libxml2 gtk`'GTK_VERSION dbus-libs - dnl libvirt is optional, used just to parse the host CPU capabilities. - ifdef(`HAVE_LIBVIRT', `libvirt-libs') dnl Run as external programs by the p2v binary. /usr/bin/ssh @@ -66,7 +64,6 @@ ifelse(DEBIAN,1, libxml2 ifelse(GTK_VERSION,2,libgtk`'GTK_VERSION`'.0-0,libgtk-`'GTK_VERSION`'-0) libdbus-1-3 - ifdef(`HAVE_LIBVIRT', `libvirt0') openssh-client qemu-utils debianutils @@ -87,7 +84,6 @@ ifelse(ARCHLINUX,1, libxml2 gtk`'GTK_VERSION dbus - ifdef(`HAVE_LIBVIRT', `libvirt') openssh qemu which @@ -110,7 +106,6 @@ ifelse(SUSE,1, libxml2 gtk`'GTK_VERSION libdbus-1-3 - ifdef(`HAVE_LIBVIRT', `libvirt-libs') qemu-tools openssh dnl /usr/bin/which is in util-linux on SUSE -- 2.12.0
Pino Toscano
2017-Mar-24 10:39 UTC
Re: [Libguestfs] [PATCH] p2v: Use lscpu instead of libvirt to get CPU information.
On Thursday, 23 March 2017 15:31:36 CET Richard W.M. Jones wrote:> Don't get the CPU information from libvirt, because including libvirt > and all dependencies in the virt-p2v ISO bloats everything. > > Instead get most of the information we need from the util-linux > program 'lscpu'. > > Unfortunately the CPU model cannot be retrieved.Theoretically, at least for Intel CPUs it could be mapped from the "Model" field; there's a table on the intel website: https://software.intel.com/en-us/articles/intel-architecture-and-processor-identification-with-cpuid-model-and-family-numbers although it is not up-to-date...> @@ -65,235 +57,166 @@ free_cpu_config (struct cpu_config *cpu) > } > > /** > - * Read flags from F</proc/cpuinfo>. > + * Get the output of lscpu as a list of (key, value) pairs (as a > + * flattened list of strings). > */ > -static void > -cpuinfo_flags (struct cpu_config *cpu) > +static char ** > +get_lscpu (void) > { > const char *cmd; > CLEANUP_PCLOSE FILE *fp = NULL; > - CLEANUP_FREE char *flag = NULL; > + CLEANUP_FREE char *line = NULL; > ssize_t len; > size_t buflen = 0; > + char **ret = NULL; > + size_t ret_size = 0; > > - /* Get the flags, one per line. */ > - cmd = "< /proc/cpuinfo " > -#if defined(__arm__) > - "grep ^Features" > -#else > - "grep ^flags" > -#endif > - " | awk '{ for (i = 3; i <= NF; ++i) { print $i }; exit }'"; > + cmd = "lscpu"; > > fp = popen (cmd, "re"); > if (fp == NULL) { > - perror ("/proc/cpuinfo"); > - return; > + perror (cmd); > + return NULL; > } > > - while (errno = 0, (len = getline (&flag, &buflen, fp)) != -1) { > - if (len > 0 && flag[len-1] == '\n') > - flag[len-1] = '\0'; > - > - if (STREQ (flag, "acpi")) > - cpu->acpi = 1; > - else if (STREQ (flag, "apic")) > - cpu->apic = 1; > - else if (STREQ (flag, "pae")) > - cpu->pae = 1; > + ret = malloc (sizeof (char *)); > + if (ret == NULL) error (EXIT_FAILURE, errno, "malloc"); > + ret[0] = NULL; > + > + while (errno = 0, (len = getline (&line, &buflen, fp)) != -1) {I know it was already in the previous version, but why the need to reset errno for each iteration? IIRC the return code of getline can be trusted to know when an error happened.> + if (vendor) { > + /* Note this mapping comes from /usr/share/libvirt/cpu_map.xml */ > + if (STREQ (vendor, "GenuineIntel")) > + cpu->vendor = strdup ("Intel"); > + else if (STREQ (vendor, "AuthenticAMD")) > + cpu->vendor = strdup ("AMD"); > + /* Currently aarch64 lscpu has no Vendor ID XXX. */ > + }How do tools such as dmidecode (use `dmidecode --type 4` to get only the processor information) or lshw (`lshw -class processor`) behave on aarch64? - dmidecode does not have a machine parseable output, but does provide the CPU model ID - lshw has both XML and JSON output, but it does not seem to provide the CPU model ID The rest of the changes would look fine otherwise. -- Pino Toscano
Richard W.M. Jones
2017-Mar-24 10:53 UTC
Re: [Libguestfs] [PATCH] p2v: Use lscpu instead of libvirt to get CPU information.
On Fri, Mar 24, 2017 at 11:39:52AM +0100, Pino Toscano wrote:> On Thursday, 23 March 2017 15:31:36 CET Richard W.M. Jones wrote: > > Don't get the CPU information from libvirt, because including libvirt > > and all dependencies in the virt-p2v ISO bloats everything. > > > > Instead get most of the information we need from the util-linux > > program 'lscpu'. > > > > Unfortunately the CPU model cannot be retrieved. > > Theoretically, at least for Intel CPUs it could be mapped from the > "Model" field; there's a table on the intel website: > https://software.intel.com/en-us/articles/intel-architecture-and-processor-identification-with-cpuid-model-and-family-numbers > although it is not up-to-date...To map it into a libvirt model (which is ultimately what we want) we'd have to use /usr/share/libvirt/cpu_map.xml, which involves some complicated mapping from the flags information. It's a shame this code is buried in libvirt and not available separately.> > - while (errno = 0, (len = getline (&flag, &buflen, fp)) != -1) { > > - if (len > 0 && flag[len-1] == '\n') > > - flag[len-1] = '\0'; > > - > > - if (STREQ (flag, "acpi")) > > - cpu->acpi = 1; > > - else if (STREQ (flag, "apic")) > > - cpu->apic = 1; > > - else if (STREQ (flag, "pae")) > > - cpu->pae = 1; > > + ret = malloc (sizeof (char *)); > > + if (ret == NULL) error (EXIT_FAILURE, errno, "malloc"); > > + ret[0] = NULL; > > + > > + while (errno = 0, (len = getline (&line, &buflen, fp)) != -1) { > > I know it was already in the previous version, but why the need to > reset errno for each iteration? IIRC the return code of getline can > be trusted to know when an error happened.I believe it's because getline returns -1 on EOF as well as error, and I want to distinguish the two cases (although I have to admit I just copied it from the old code).> > + if (vendor) { > > + /* Note this mapping comes from /usr/share/libvirt/cpu_map.xml */ > > + if (STREQ (vendor, "GenuineIntel")) > > + cpu->vendor = strdup ("Intel"); > > + else if (STREQ (vendor, "AuthenticAMD")) > > + cpu->vendor = strdup ("AMD"); > > + /* Currently aarch64 lscpu has no Vendor ID XXX. */ > > + } > > How do tools such as dmidecode (use `dmidecode --type 4` to get only > the processor information) or lshw (`lshw -class processor`) behave on > aarch64? > - dmidecode does not have a machine parseable output, but does provide > the CPU model ID > - lshw has both XML and JSON output, but it does not seem to provide the > CPU model ID > > The rest of the changes would look fine otherwise.I think as above that we need the libvirt CPU model specifically, and not just any old CPU model. On my x86_64 laptop, dmidecode returns "Core i7" (not "Broadwell"). On aarch64 it depends on whether we're using ACPI or DT for boot. For ACPI, dmidecode information is available, but inaccurate ("Family: ARM" ... hmm). lshw has the same info as /proc/cpuinfo which is useless for us. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-p2v converts physical machines to virtual machines. Boot with a live CD or over the network (PXE) and turn machines into KVM guests. http://libguestfs.org/virt-v2v
Possibly Parallel Threads
- [PATCH] p2v: Use lscpu instead of libvirt to get CPU information.
- [PATCH 1/4] p2v: Pass host CPU details to virt-v2v.
- Command line mp3 player
- [PATCH] p2v: Calculate offset of the Real Time Clock from UTC.
- [PATCH 0/4] Pass CPU vendor, model and topology from source to target.