dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 0/8] enable Cache QoS Monitoring (CQM) feature
From: Dongxiao Xu <dongxiao.xu@intel.com> Future generations of Intel Xeon processor may offer monitoring capability in each logical processor to measure specific quality-of-service metric, for example, the Cache QoS Monitoring to get L3 cache occupancy. Detailed information please refer to Intel SDM chapter 17.14. Cache QoS Monitoring provides a layer of abstraction between applications and logical processors through the use of Resource Monitoring IDs (RMIDs). In Xen design, each guest in the system can be assigned an RMID independently, while RMID=0 is reserved for monitoring domains that doesn''t enable CQM service. When any of the domain''s vcpu is scheduled on a logical processor, the domain''s RMID will be activated by programming the value into one specific MSR, and when the vcpu is scheduled out, a RMID=0 will be programmed into that MSR. The Cache QoS Hardware tracks cache utilization of memory accesses according to the RMIDs and reports monitored data via a counter register. With this solution, we can get the knowledge how much L3 cache is used by a certain guest. To attach CQM service to a certain guest, two approaches are provided: 1) Create the guest with "pqos_cqm=1" set in configuration file. 2) Use "xl pqos-attach cqm domid" for a running guest. To detached CQM service from a guest, users can: 1) Use "xl pqos-detach cqm domid" for a running guest. 2) Also destroying a guest will detach the CQM service. To get the L3 cache usage, users can use the command of: $ xl pqos-list cqm (domid) The below data is just an example showing how the CQM related data is exposed to end user. [root@localhost]# xl pqos-list cqm RMID count 56 RMID available 53 Name ID SocketID L3C_Usage SocketID L3C_Usage Domain-0 0 0 20127744 1 25231360 ExampleHVMDomain 1 0 3211264 1 10551296 Dongxiao Xu (8): x86: detect and initialize Cache QoS Monitoring feature x86: handle CQM resource when creating/destroying guests tools: handle CQM resource when creating/destroying guests x86: dynamically attach/detach CQM service for a guest tools: dynamically attach/detach CQM service for a guest x86: get per domain CQM information tools: get per domain CQM information x86: enable CQM monitoring for each domain RMID tools/libxc/xc_domain.c | 51 +++++++++ tools/libxc/xenctrl.h | 7 ++ tools/libxl/Makefile | 3 +- tools/libxl/libxl.h | 8 ++ tools/libxl/libxl_create.c | 3 + tools/libxl/libxl_pqos.c | 100 +++++++++++++++++ tools/libxl/libxl_types.idl | 1 + tools/libxl/xl.h | 3 + tools/libxl/xl_cmdimpl.c | 129 ++++++++++++++++++++++ tools/libxl/xl_cmdtable.c | 15 +++ xen/arch/x86/Makefile | 1 + xen/arch/x86/cpu/intel.c | 6 + xen/arch/x86/domain.c | 17 +++ xen/arch/x86/domctl.c | 66 +++++++++++ xen/arch/x86/pqos.c | 226 ++++++++++++++++++++++++++++++++++++++ xen/arch/x86/setup.c | 3 + xen/arch/x86/sysctl.c | 64 +++++++++++ xen/common/domctl.c | 5 +- xen/include/asm-x86/cpufeature.h | 1 + xen/include/asm-x86/domain.h | 2 + xen/include/asm-x86/msr-index.h | 5 + xen/include/asm-x86/pqos.h | 56 ++++++++++ xen/include/public/domctl.h | 32 ++++++ xen/include/public/sysctl.h | 23 ++++ xen/include/xen/sched.h | 3 + 25 files changed, 828 insertions(+), 2 deletions(-) create mode 100644 tools/libxl/libxl_pqos.c create mode 100644 xen/arch/x86/pqos.c create mode 100644 xen/include/asm-x86/pqos.h -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
From: Dongxiao Xu <dongxiao.xu@intel.com> Detect platform QoS feature status and enumerate the resource types, one of which is to monitor the L3 cache occupancy. Also introduce a Xen grub command line parameter to control the QoS feature status globally. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- xen/arch/x86/Makefile | 1 + xen/arch/x86/cpu/intel.c | 6 +++ xen/arch/x86/pqos.c | 89 ++++++++++++++++++++++++++++++++++++++ xen/arch/x86/setup.c | 3 ++ xen/include/asm-x86/cpufeature.h | 1 + xen/include/asm-x86/pqos.h | 37 ++++++++++++++++ 6 files changed, 137 insertions(+) create mode 100644 xen/arch/x86/pqos.c create mode 100644 xen/include/asm-x86/pqos.h diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile index d502bdf..54962e0 100644 --- a/xen/arch/x86/Makefile +++ b/xen/arch/x86/Makefile @@ -58,6 +58,7 @@ obj-y += crash.o obj-y += tboot.o obj-y += hpet.o obj-y += xstate.o +obj-y += pqos.o obj-$(crash_debug) += gdbstub.o diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c index 27fe762..f0d83ea 100644 --- a/xen/arch/x86/cpu/intel.c +++ b/xen/arch/x86/cpu/intel.c @@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c) ( c->cpuid_level >= 0x00000006 ) && ( cpuid_eax(0x00000006) & (1u<<2) ) ) set_bit(X86_FEATURE_ARAT, c->x86_capability); + + /* Check platform QoS monitoring capability */ + if ((c->cpuid_level >= 0x00000007) && + (cpuid_ebx(0x00000007) & (1u<<12))) + set_bit(X86_FEATURE_QOSM, c->x86_capability); + } static struct cpu_dev intel_cpu_dev __cpuinitdata = { diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c new file mode 100644 index 0000000..617cce1 --- /dev/null +++ b/xen/arch/x86/pqos.c @@ -0,0 +1,89 @@ +/* + * pqos.c: Platform QoS related service for guest. + * + * Copyright (c) 2013, Intel Corporation + * Author: Jiongxi Li <jiongxi.li@intel.com> + * Author: Dongxiao Xu <dongxiao.xu@intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + */ +#include <asm/processor.h> +#include <xen/init.h> +#include <asm/pqos.h> + +static bool_t pqos_enabled = 1; +boolean_param("pqos", pqos_enabled); + +unsigned int cqm_res_count = 0; +unsigned int cqm_upscaling_factor = 0; +bool_t cqm_enabled = 0; +struct cqm_res_struct *cqm_res_array = NULL; + +static void init_cqm(void) +{ + unsigned int eax, edx; + unsigned int max_cqm_rmid; + + cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid, &edx); + if ( !(edx & QOS_MONITOR_EVTID_L3) ) + return; + + cqm_res_count = max_cqm_rmid + 1; + + cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count); + if ( !cqm_res_array ) + { + cqm_res_count = 0; + return; + } + + cqm_enabled = 1; + + /* Reserve RMID 0 for all domains not being monitored */ + cqm_res_array[0].inuse = 1; + + printk("Cache QoS Monitoring Enabled.\n"); +} + +static void init_qos_monitor(void) +{ + unsigned int qm_features; + unsigned int eax, ebx, ecx; + + if ( !(boot_cpu_has(X86_FEATURE_QOSM)) ) + return; + + cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features); + + if ( qm_features & QOS_MONITOR_TYPE_L3 ) + init_cqm(); +} + +void init_platform_qos(void) +{ + if ( !pqos_enabled ) + return; + + init_qos_monitor(); +} + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c index 5bf4ee0..95418e4 100644 --- a/xen/arch/x86/setup.c +++ b/xen/arch/x86/setup.c @@ -48,6 +48,7 @@ #include <asm/setup.h> #include <xen/cpu.h> #include <asm/nmi.h> +#include <asm/pqos.h> /* opt_nosmp: If true, secondary processors are ignored. */ static bool_t __initdata opt_nosmp; @@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p) domain_unpause_by_systemcontroller(dom0); + init_platform_qos(); + reset_stack_and_jump(init_done); } diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h index 1cfaf94..ca59668 100644 --- a/xen/include/asm-x86/cpufeature.h +++ b/xen/include/asm-x86/cpufeature.h @@ -147,6 +147,7 @@ #define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */ #define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context ID */ #define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional Memory */ +#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring capability */ #define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */ #define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access Prevention */ diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h new file mode 100644 index 0000000..2184ea9 --- /dev/null +++ b/xen/include/asm-x86/pqos.h @@ -0,0 +1,37 @@ +/* + * pqos.h: Platform QoS related service for guest. + * + * Copyright (c) 2013, Intel Corporation + * Author: Jiongxi Li <jiongxi.li@intel.com> + * Author: Dongxiao Xu <dongxiao.xu@intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + */ +#ifndef ASM_PQOS_H +#define ASM_PQOS_H + +/* QoS Resource Type Enumeration */ +#define QOS_MONITOR_TYPE_L3 0x2 + +/* QoS Monitoring Event ID */ +#define QOS_MONITOR_EVTID_L3 0x1 + +struct cqm_res_struct { + bool_t inuse; + uint16_t domain_id; +}; + +void init_platform_qos(void); + +#endif -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 2/8] x86: handle CQM resource when creating/destroying guests
From: Dongxiao Xu <dongxiao.xu@intel.com> Allocate an RMID for a guest when it is created. This per-guest RMID will be used to monitor Cache QoS related data. The RMID will be relinquished when guest is destroyed. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- xen/arch/x86/domain.c | 12 +++++++++ xen/arch/x86/pqos.c | 58 ++++++++++++++++++++++++++++++++++++++++++ xen/common/domctl.c | 5 +++- xen/include/asm-x86/domain.h | 2 ++ xen/include/asm-x86/pqos.h | 4 +++ xen/include/public/domctl.h | 3 +++ xen/include/xen/sched.h | 3 +++ 7 files changed, 86 insertions(+), 1 deletion(-) diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index a3868f9..5f920cd 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -60,6 +60,7 @@ #include <xen/numa.h> #include <xen/iommu.h> #include <compat/vcpu.h> +#include <asm/pqos.h> DEFINE_PER_CPU(struct vcpu *, curr_vcpu); DEFINE_PER_CPU(unsigned long, cr4); @@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags) tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0); spin_lock_init(&d->arch.vtsc_lock); + /* Allocate CQM RMID for guest */ + if ( !!(domcr_flags & DOMCRF_pqos_cqm) ) + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id); + else if ( system_support_cqm() ) + d->arch.pqos_cqm_rmid = 0; + else + d->arch.pqos_cqm_rmid = -1; + return 0; fail: @@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d) free_xenheap_page(d->shared_info); cleanup_domain_irq_mapping(d); + + free_cqm_resource(d->domain_id); + d->arch.pqos_cqm_rmid = 0; } unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long guest_cr4) diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c index 617cce1..895d892 100644 --- a/xen/arch/x86/pqos.c +++ b/xen/arch/x86/pqos.c @@ -20,6 +20,7 @@ */ #include <asm/processor.h> #include <xen/init.h> +#include <xen/spinlock.h> #include <asm/pqos.h> static bool_t pqos_enabled = 1; @@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0; unsigned int cqm_upscaling_factor = 0; bool_t cqm_enabled = 0; struct cqm_res_struct *cqm_res_array = NULL; +static DEFINE_SPINLOCK(cqm_lock); static void init_cqm(void) { @@ -78,6 +80,62 @@ void init_platform_qos(void) init_qos_monitor(); } +bool_t system_support_cqm(void) +{ + return cqm_enabled; +} + +int alloc_cqm_resource(domid_t domain_id) +{ + int i, rmid = -1; + unsigned long flags; + + if ( !system_support_cqm() ) + goto out; + + spin_lock_irqsave(&cqm_lock, flags); + for ( i = 0; i < cqm_res_count; i++ ) + { + if ( cqm_res_array[i].inuse) + continue; + + rmid = i; + cqm_res_array[rmid].inuse = 1; + cqm_res_array[rmid].domain_id = domain_id; + break; + } + spin_unlock_irqrestore(&cqm_lock, flags); + + /* No CQM RMID available, assign RMID=0 by default */ + if ( rmid == -1 ) + rmid = 0; + +out: + return rmid; +} + +void free_cqm_resource(domid_t domain_id) +{ + int rmid; + unsigned long flags; + + if ( !system_support_cqm() ) + return; + + spin_lock_irqsave(&cqm_lock, flags); + /* CQM RMID 0 is reserved, enumerate from 1 */ + for ( rmid = 1; rmid < cqm_res_count; rmid++ ) + { + if ( cqm_res_array[rmid].domain_id != domain_id ) + continue; + + cqm_res_array[rmid].inuse = 0; + cqm_res_array[rmid].domain_id = 0; + break; + } + spin_unlock_irqrestore(&cqm_lock, flags); +} + /* * Local variables: * mode: C diff --git a/xen/common/domctl.c b/xen/common/domctl.c index 904d27b..1c2e320 100644 --- a/xen/common/domctl.c +++ b/xen/common/domctl.c @@ -425,7 +425,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) | XEN_DOMCTL_CDF_pvh_guest | XEN_DOMCTL_CDF_hap | XEN_DOMCTL_CDF_s3_integrity - | XEN_DOMCTL_CDF_oos_off)) ) + | XEN_DOMCTL_CDF_oos_off + | XEN_DOMCTL_CDF_pqos_cqm)) ) break; dom = op->domain; @@ -467,6 +468,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) domcr_flags |= DOMCRF_s3_integrity; if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off ) domcr_flags |= DOMCRF_oos_off; + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm ) + domcr_flags |= DOMCRF_pqos_cqm; d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref); if ( IS_ERR(d) ) diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h index 9d39061..b0479aa 100644 --- a/xen/include/asm-x86/domain.h +++ b/xen/include/asm-x86/domain.h @@ -313,6 +313,8 @@ struct arch_domain spinlock_t e820_lock; struct e820entry *e820; unsigned int nr_e820; + + int pqos_cqm_rmid; /* CQM RMID assigned to the domain */ } __cacheline_aligned; #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list)) diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h index 2184ea9..7e32fa5 100644 --- a/xen/include/asm-x86/pqos.h +++ b/xen/include/asm-x86/pqos.h @@ -34,4 +34,8 @@ struct cqm_res_struct { void init_platform_qos(void); +bool_t system_support_cqm(void); +int alloc_cqm_resource(domid_t); +void free_cqm_resource(domid_t); + #endif diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h index 01a3652..47a850a 100644 --- a/xen/include/public/domctl.h +++ b/xen/include/public/domctl.h @@ -62,6 +62,9 @@ struct xen_domctl_createdomain { /* Is this a PVH guest (as opposed to an HVM or PV guest)? */ #define _XEN_DOMCTL_CDF_pvh_guest 4 #define XEN_DOMCTL_CDF_pvh_guest (1U<<_XEN_DOMCTL_CDF_pvh_guest) + /* Enable pqos-cqm? */ +#define _XEN_DOMCTL_CDF_pqos_cqm 5 +#define XEN_DOMCTL_CDF_pqos_cqm (1U<<_XEN_DOMCTL_CDF_pqos_cqm) uint32_t flags; }; typedef struct xen_domctl_createdomain xen_domctl_createdomain_t; diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h index cbdf377..3a42656 100644 --- a/xen/include/xen/sched.h +++ b/xen/include/xen/sched.h @@ -507,6 +507,9 @@ struct domain *domain_create( /* DOMCRF_pvh: Create PV domain in HVM container. */ #define _DOMCRF_pvh 5 #define DOMCRF_pvh (1U<<_DOMCRF_pvh) + /* DOMCRF_pqos_cqm: Create a domain with CQM support */ +#define _DOMCRF_pqos_cqm 6 +#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm) /* * rcu_lock_domain_by_id() is more efficient than get_domain_by_id(). -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 3/8] tools: handle CQM resource when creating/destroying guests
From: Dongxiao Xu <dongxiao.xu@intel.com> If a guest is created with parameter "pqos_cqm" set to 1, an RMID will be allocated to the guest. This RMID will be relinquished when guest is destroyed. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- tools/libxl/libxl_create.c | 3 +++ tools/libxl/libxl_types.idl | 1 + tools/libxl/xl_cmdimpl.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c index 5e9cdcc..454c69d 100644 --- a/tools/libxl/libxl_create.c +++ b/tools/libxl/libxl_create.c @@ -40,6 +40,8 @@ int libxl__domain_create_info_setdefault(libxl__gc *gc, libxl_defbool_setdefault(&c_info->run_hotplug_scripts, true); + libxl_defbool_setdefault(&c_info->pqos_cqm, false); + return 0; } @@ -454,6 +456,7 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_create_info *info, } flags |= XEN_DOMCTL_CDF_hap; } + flags |= libxl_defbool_val(info->pqos_cqm) ? XEN_DOMCTL_CDF_pqos_cqm : 0; *domid = -1; /* Ultimately, handle is an array of 16 uint8_t, same as uuid */ diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index de5bac3..22688d8 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -275,6 +275,7 @@ libxl_domain_create_info = Struct("domain_create_info",[ ("poolid", uint32), ("run_hotplug_scripts",libxl_defbool), ("pvh", libxl_defbool), + ("pqos_cqm", libxl_defbool), ], dir=DIR_IN) libxl_domain_restore_params = Struct("domain_restore_params", [ diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 8690ec7..84a604f 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -670,6 +670,8 @@ static void parse_config_data(const char *config_source, exit(1); } + xlu_cfg_get_defbool(config, "pqos_cqm", &c_info->pqos_cqm, 0); + libxl_domain_build_info_init_type(b_info, c_info->type); if (blkdev_start) b_info->blkdev_start = strdup(blkdev_start); -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
From: Dongxiao Xu <dongxiao.xu@intel.com> Add hypervisor side support for dynamically attach and detach CQM services for a certain guest. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- xen/arch/x86/domctl.c | 66 +++++++++++++++++++++++++++++++++++++++++++ xen/include/public/domctl.h | 15 ++++++++++ 2 files changed, 81 insertions(+) diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c index f7e4586..df6a373 100644 --- a/xen/arch/x86/domctl.c +++ b/xen/arch/x86/domctl.c @@ -35,6 +35,7 @@ #include <asm/mem_sharing.h> #include <asm/xstate.h> #include <asm/debugger.h> +#include <asm/pqos.h> static int gdbsx_guest_mem_io( domid_t domid, struct xen_domctl_gdbsx_memio *iop) @@ -1223,6 +1224,71 @@ long arch_do_domctl( } break; + case XEN_DOMCTL_attach_pqos: + { + struct domain *d; + + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) + { + ret = -EBUSY; + break; + } + + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) + { + if ( !system_support_cqm() ) + ret = -ENODEV; + else if ( d->arch.pqos_cqm_rmid > 0 ) + ret = -EINVAL; + else + { + ret = 0; + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id); + if ( d->arch.pqos_cqm_rmid <= 0 ) + { + /* set to default value */ + d->arch.pqos_cqm_rmid = 0; + ret = -ENODEV; + } + } + } + else + ret = -EINVAL; + + rcu_unlock_domain(d); + } + break; + + case XEN_DOMCTL_detach_pqos: + { + struct domain *d; + + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) + { + ret = -EBUSY; + break; + } + + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) + { + if ( !system_support_cqm() ) + ret = -ENODEV; + else if ( d->arch.pqos_cqm_rmid > 0 ) + { + free_cqm_resource(d->domain_id); + d->arch.pqos_cqm_rmid = 0; + ret = 0; + } + else + ret = -EINVAL; + } + else + ret = -EINVAL; + + rcu_unlock_domain(d); + } + break; + default: ret = iommu_do_domctl(domctl, d, u_domctl); break; diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h index 47a850a..4fe21db 100644 --- a/xen/include/public/domctl.h +++ b/xen/include/public/domctl.h @@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn { typedef struct xen_domctl_set_max_evtchn xen_domctl_set_max_evtchn_t; DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t); +/* XEN_DOMCTL_attach_pqos */ +/* XEN_DOMCTL_detach_pqos */ +struct xen_domctl_qos_resource { + /* Attach or detach flag for cqm */ +#define _XEN_DOMCTL_ADF_pqos_cqm 0 +#define XEN_DOMCTL_ADF_pqos_cqm (1U<<_XEN_DOMCTL_ADF_pqos_cqm) + uint32_t flags; +}; +typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; +DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); + struct xen_domctl { uint32_t cmd; #define XEN_DOMCTL_createdomain 1 @@ -941,6 +952,9 @@ struct xen_domctl { #define XEN_DOMCTL_setnodeaffinity 68 #define XEN_DOMCTL_getnodeaffinity 69 #define XEN_DOMCTL_set_max_evtchn 70 +#define XEN_DOMCTL_attach_pqos 71 +#define XEN_DOMCTL_detach_pqos 72 +#define XEN_DOMCTL_list_pqos 73 #define XEN_DOMCTL_gdbsx_guestmemio 1000 #define XEN_DOMCTL_gdbsx_pausevcpu 1001 #define XEN_DOMCTL_gdbsx_unpausevcpu 1002 @@ -1001,6 +1015,7 @@ struct xen_domctl { struct xen_domctl_set_broken_page_p2m set_broken_page_p2m; struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu; struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; + struct xen_domctl_qos_resource qos_res; uint8_t pad[128]; } u; }; -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 5/8] tools: dynamically attach/detach CQM service for a guest
From: Dongxiao Xu <dongxiao.xu@intel.com> Add two new xl instruction for attaching and detaching CQM services for a running guest. Above "qos_res" is the monitoring resource type, and current supported type is "cqm". Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- tools/libxc/xc_domain.c | 18 ++++++++++ tools/libxc/xenctrl.h | 2 ++ tools/libxl/Makefile | 3 +- tools/libxl/libxl.h | 3 ++ tools/libxl/libxl_pqos.c | 86 +++++++++++++++++++++++++++++++++++++++++++++ tools/libxl/xl.h | 2 ++ tools/libxl/xl_cmdimpl.c | 36 +++++++++++++++++++ tools/libxl/xl_cmdtable.c | 10 ++++++ 8 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 tools/libxl/libxl_pqos.c diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c index 1ccafc5..85c2d4d 100644 --- a/tools/libxc/xc_domain.c +++ b/tools/libxc/xc_domain.c @@ -1776,6 +1776,24 @@ int xc_domain_set_max_evtchn(xc_interface *xch, uint32_t domid, return do_domctl(xch, &domctl); } +int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags) +{ + DECLARE_DOMCTL; + domctl.cmd = XEN_DOMCTL_attach_pqos; + domctl.domain = (domid_t)domid; + domctl.u.qos_res.flags = flags; + return do_domctl(xch, &domctl); +} + +int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags) +{ + DECLARE_DOMCTL; + domctl.cmd = XEN_DOMCTL_detach_pqos; + domctl.domain = (domid_t)domid; + domctl.u.qos_res.flags = flags; + return do_domctl(xch, &domctl); +} + /* * Local variables: * mode: C diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h index 4ac6b8a..a57f147 100644 --- a/tools/libxc/xenctrl.h +++ b/tools/libxc/xenctrl.h @@ -2395,4 +2395,6 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch, */ int xc_kexec_unload(xc_interface *xch, int type); +int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags); +int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags); #endif /* XENCTRL_H */ diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index cf214bb..35f0b97 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -74,7 +74,8 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \ libxl_internal.o libxl_utils.o libxl_uuid.o \ libxl_json.o libxl_aoutils.o libxl_numa.o \ libxl_save_callout.o _libxl_save_msgs_callout.o \ - libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y) + libxl_qmp.o libxl_event.o libxl_fork.o libxl_pqos.o \ + $(LIBXL_OBJS-y) LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o $(LIBXL_OBJS): CFLAGS += $(CFLAGS_LIBXL) -include $(XEN_ROOT)/tools/config.h diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index c7dceda..a9a506f 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -1051,6 +1051,9 @@ int libxl_flask_getenforce(libxl_ctx *ctx); int libxl_flask_setenforce(libxl_ctx *ctx, int mode); int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size); +int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres); +int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres); + /* misc */ /* Each of these sets or clears the flag according to whether the diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c new file mode 100644 index 0000000..6d8013a --- /dev/null +++ b/tools/libxl/libxl_pqos.c @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2013 Intel Corporation + * Author Jiongxi Li <jiongxi.li@intel.com> + * Author Dongxiao Xu <dongxiao.xu@intel.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ +#include "libxl_internal.h" + +int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres) +{ + int rc; + int flags = 0; + + if (!strcmp(qosres, "cqm")) + flags |= XEN_DOMCTL_ADF_pqos_cqm; + else { + rc = -EINVAL; + fprintf(stderr, "Invalid QoS resource type! " + "Supported types are: \"cqm\".\n"); + return rc; + } + + rc = xc_domain_pqos_attach(ctx->xch, domid, flags); + if (rc < 0) { + fprintf(stderr, "Failed to attach CQM service to domain %d. ", domid); + if (errno == ENODEV) + fprintf(stderr, "CQM not supported, or no free CQM RMID available.\n"); + else if (errno == EINVAL) + fprintf(stderr, "Domain %d is already attached with CQM service.\n", domid); + else if (errno == ESRCH) + fprintf(stderr, "Is Domain %d a valid domain?\n", domid); + else + fprintf(stderr, "errno: %d\n", errno); + } + + return rc; +} + +int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres) +{ + int rc; + int flags = 0; + + if (!strcmp(qosres, "cqm")) + flags |= XEN_DOMCTL_ADF_pqos_cqm; + else { + rc = -EINVAL; + fprintf(stderr, "Invalid QoS resource type! " + "Supported types are: \"cqm\".\n"); + return rc; + } + + rc = xc_domain_pqos_detach(ctx->xch, domid, flags); + if (rc < 0) { + fprintf(stderr, "Failed to detach CQM service to domain %d. ", domid); + if (errno == ENODEV) + fprintf(stderr, "CQM not supported.\n"); + else if (errno == EINVAL) + fprintf(stderr, "Domain %d is not attached with CQM service.\n", domid); + else if (errno == ESRCH) + fprintf(stderr, "Is Domain %d a valid domain?\n", domid); + else + fprintf(stderr, "errno: %d\n", errno); + } + + return rc; +} + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h index e005c39..78738b9 100644 --- a/tools/libxl/xl.h +++ b/tools/libxl/xl.h @@ -105,6 +105,8 @@ int main_getenforce(int argc, char **argv); int main_setenforce(int argc, char **argv); int main_loadpolicy(int argc, char **argv); int main_remus(int argc, char **argv); +int main_pqosattach(int argc, char **argv); +int main_pqosdetach(int argc, char **argv); void help(const char *command); diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 84a604f..1a12e8f 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -7195,6 +7195,42 @@ int main_remus(int argc, char **argv) return -ERROR_FAIL; } +int main_pqosattach(int argc, char **argv) +{ + uint32_t domid; + int opt, rc; + const char *qosres = NULL; + + SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-attach", 2) { + /* No options */ + } + + qosres = argv[optind]; + domid = find_domain(argv[optind + 1]); + + rc = libxl_pqos_attach(ctx, domid, qosres); + + return rc; +} + +int main_pqosdetach(int argc, char **argv) +{ + uint32_t domid; + int opt, rc; + const char *qosres = NULL; + + SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-detach", 2) { + /* No options */ + } + + qosres = argv[optind]; + domid = find_domain(argv[optind + 1]); + + rc = libxl_pqos_detach(ctx, domid, qosres); + + return rc; +} + /* * Local variables: * mode: C diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index 326a660..02a2572 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -488,6 +488,16 @@ struct cmd_spec cmd_table[] = { " of the domain." }, + { "pqos-attach", + &main_pqosattach, 0, 1, + "Allocate and map qos resource", + "<Resource> <Domain>", + }, + { "pqos-detach", + &main_pqosdetach, 0, 1, + "Reliquish qos resource", + "<Resource> <Domain>", + }, }; int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec); -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 6/8] x86: get per domain CQM information
From: Dongxiao Xu <dongxiao.xu@intel.com> Retrive CQM information for certain domain, which reflects the L3 cache occupancy for a socket. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- xen/arch/x86/pqos.c | 60 ++++++++++++++++++++++++++++++++++++ xen/arch/x86/sysctl.c | 64 +++++++++++++++++++++++++++++++++++++++ xen/include/asm-x86/msr-index.h | 4 +++ xen/include/asm-x86/pqos.h | 14 +++++++++ xen/include/public/domctl.h | 14 +++++++++ xen/include/public/sysctl.h | 23 ++++++++++++++ 6 files changed, 179 insertions(+) diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c index 895d892..3699efe 100644 --- a/xen/arch/x86/pqos.c +++ b/xen/arch/x86/pqos.c @@ -19,13 +19,30 @@ * Place - Suite 330, Boston, MA 02111-1307 USA. */ #include <asm/processor.h> +#include <asm/msr.h> +#include <xen/cpumask.h> #include <xen/init.h> #include <xen/spinlock.h> +#include <public/domctl.h> #include <asm/pqos.h> static bool_t pqos_enabled = 1; boolean_param("pqos", pqos_enabled); +static void read_qm_data(void *arg) +{ + struct qm_element *qm_element = arg; + + wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid, qm_element->rmid); + rdmsrl(MSR_IA32_QMC, qm_element->qm_data); +} + +static void get_generic_qm_info(struct qm_element *qm_element) +{ + int cpu = qm_element->cpu; + on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1); +} + unsigned int cqm_res_count = 0; unsigned int cqm_upscaling_factor = 0; bool_t cqm_enabled = 0; @@ -85,6 +102,25 @@ bool_t system_support_cqm(void) return cqm_enabled; } +unsigned int get_cqm_count(void) +{ + return cqm_res_count; +} + +unsigned int get_cqm_avail(void) +{ + unsigned int cqm_avail = 0; + int i; + + for (i = 0; i < cqm_res_count; i++) + { + if ( !cqm_res_array[i].inuse ) + cqm_avail++; + } + + return cqm_avail; +} + int alloc_cqm_resource(domid_t domain_id) { int i, rmid = -1; @@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id) spin_unlock_irqrestore(&cqm_lock, flags); } +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, + struct xen_domctl_getdomcqminfo *info) +{ + struct qm_element element; + int cpu, i; + + for_each_cpu ( cpu, &cpu_cqmdata_map ) + { + element.cpu = cpu; + element.rmid = rmid; + element.evtid = QOS_MONITOR_EVTID_L3; + + get_generic_qm_info(&element); + + i = cpu_to_socket(cpu); + info->socket_cqmdata[i].valid + (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1; + if ( info->socket_cqmdata[i].valid ) + info->socket_cqmdata[i].l3c_occupancy = element.qm_data * cqm_upscaling_factor; + else + info->socket_cqmdata[i].l3c_occupancy = 0; + } +} + /* * Local variables: * mode: C diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c index 15d4b91..a779fdc 100644 --- a/xen/arch/x86/sysctl.c +++ b/xen/arch/x86/sysctl.c @@ -28,6 +28,7 @@ #include <xen/nodemask.h> #include <xen/cpu.h> #include <xsm/xsm.h> +#include <asm/pqos.h> #define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0) @@ -101,6 +102,69 @@ long arch_do_sysctl( } break; + case XEN_SYSCTL_getdomcqminfolist: + { + struct domain *d; + struct xen_domctl_getdomcqminfo info; + uint32_t resource_count; + uint32_t resource_avail; + uint32_t num_domains = 0; + cpumask_t cpu_cqmdata_map; + DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS); + int cpu; + + if ( !system_support_cqm() ) + { + ret = -EFAULT; + break; + } + + resource_count = get_cqm_count(); + resource_avail = get_cqm_avail(); + + cpumask_clear(&cpu_cqmdata_map); + bitmap_zero(sockets, QOS_MAX_SOCKETS); + for_each_online_cpu(cpu) + { + int i = cpu_to_socket(cpu); + if ( test_and_set_bit(i, sockets) ) + continue; + cpumask_set_cpu(cpu, &cpu_cqmdata_map); + } + + rcu_read_lock(&domlist_read_lock); + for_each_domain ( d ) + { + if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain ) + continue; + if ( num_domains == sysctl->u.getdomaininfolist.max_domains ) + break; + if ( d->arch.pqos_cqm_rmid <= 0 ) + continue; + memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo)); + info.domain = d->domain_id; + get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map, &info); + + if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer, + num_domains, &info, 1) ) + { + ret = -EFAULT; + break; + } + + num_domains++; + } + rcu_read_unlock(&domlist_read_lock); + + sysctl->u.getdomcqminfolist.num_domains = num_domains; + sysctl->u.getdomcqminfolist.resource_count = resource_count; + sysctl->u.getdomcqminfolist.resource_avail = resource_avail; + + if ( copy_to_guest(u_sysctl, sysctl, 1) ) + ret = -EFAULT; + } + break; + default: ret = -ENOSYS; break; diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h index e597a28..46ef165 100644 --- a/xen/include/asm-x86/msr-index.h +++ b/xen/include/asm-x86/msr-index.h @@ -488,4 +488,8 @@ /* Geode defined MSRs */ #define MSR_GEODE_BUSCONT_CONF0 0x00001900 +/* Platform QoS register */ +#define MSR_IA32_QOSEVTSEL 0x00000c8d +#define MSR_IA32_QMC 0x00000c8e + #endif /* __ASM_MSR_INDEX_H */ diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h index 7e32fa5..6d1b1e8 100644 --- a/xen/include/asm-x86/pqos.h +++ b/xen/include/asm-x86/pqos.h @@ -27,15 +27,29 @@ /* QoS Monitoring Event ID */ #define QOS_MONITOR_EVTID_L3 0x1 +/* IA32_QM_CTR */ +#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62) + struct cqm_res_struct { bool_t inuse; uint16_t domain_id; }; +struct qm_element { + uint16_t cpu; + uint32_t rmid; + uint8_t evtid; + uint64_t qm_data; +}; + void init_platform_qos(void); bool_t system_support_cqm(void); int alloc_cqm_resource(domid_t); void free_cqm_resource(domid_t); +unsigned int get_cqm_count(void); +unsigned int get_cqm_avail(void); +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, + struct xen_domctl_getdomcqminfo *info); #endif diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h index 4fe21db..bdefc83 100644 --- a/xen/include/public/domctl.h +++ b/xen/include/public/domctl.h @@ -883,6 +883,20 @@ struct xen_domctl_qos_resource { typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); +struct xen_socket_cqmdata { + uint8_t valid; + uint64_t l3c_occupancy; +}; + +struct xen_domctl_getdomcqminfo { + /* OUT variables. */ + domid_t domain; +#define QOS_MAX_SOCKETS 128 + struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS]; +}; +typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t; +DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t); + struct xen_domctl { uint32_t cmd; #define XEN_DOMCTL_createdomain 1 diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h index 8437d31..91f206e 100644 --- a/xen/include/public/sysctl.h +++ b/xen/include/public/sysctl.h @@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op { typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t; DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t); +struct xen_sysctl_getcqminfo +{ + uint32_t resource_count; + uint32_t resource_avail; + struct xen_domctl_getdomcqminfo *dom_cqminfo; +}; +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t; + /* XEN_SYSCTL_getdomaininfolist */ struct xen_sysctl_getdomaininfolist { /* IN variables. */ @@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op { typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t; DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t); +/* XEN_SYSCTL_getdomcqminfolist */ +struct xen_sysctl_getdomcqminfolist { + /* IN variables. */ + domid_t first_domain; + uint32_t max_domains; + XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer; + /* OUT variables. */ + uint32_t num_domains; + uint32_t resource_count; + uint32_t resource_avail; +}; +typedef struct xen_sysctl_getdomcqminfolist xen_sysctl_getdomcqminfolist_t; +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t); struct xen_sysctl { uint32_t cmd; @@ -654,6 +675,7 @@ struct xen_sysctl { #define XEN_SYSCTL_cpupool_op 18 #define XEN_SYSCTL_scheduler_op 19 #define XEN_SYSCTL_coverage_op 20 +#define XEN_SYSCTL_getdomcqminfolist 21 uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */ union { struct xen_sysctl_readconsole readconsole; @@ -675,6 +697,7 @@ struct xen_sysctl { struct xen_sysctl_cpupool_op cpupool_op; struct xen_sysctl_scheduler_op scheduler_op; struct xen_sysctl_coverage_op coverage_op; + struct xen_sysctl_getdomcqminfolist getdomcqminfolist; uint8_t pad[128]; } u; }; -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 7/8] tools: get per domain CQM information
From: Dongxiao Xu <dongxiao.xu@intel.com> Retrive CQM information for certain domain, which reflects the L3 cache occupancy for a socket. The xl command is "xl pqos-list cqm (domid)". Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- tools/libxc/xc_domain.c | 33 ++++++++++++++++ tools/libxc/xenctrl.h | 5 +++ tools/libxl/libxl.h | 5 +++ tools/libxl/libxl_pqos.c | 14 +++++++ tools/libxl/xl.h | 1 + tools/libxl/xl_cmdimpl.c | 91 +++++++++++++++++++++++++++++++++++++++++++++ tools/libxl/xl_cmdtable.c | 5 +++ 7 files changed, 154 insertions(+) diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c index 85c2d4d..96a1d50 100644 --- a/tools/libxc/xc_domain.c +++ b/tools/libxc/xc_domain.c @@ -1794,6 +1794,39 @@ int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags) return do_domctl(xch, &domctl); } +int xc_domain_getcqminfolist(xc_interface *xch, + uint32_t first_domain, + unsigned int max_domains, + sysctl_cqminfo_t *info) +{ + int ret = 0; + xen_domctl_getdomcqminfo_t *d_info = info->dom_cqminfo; + DECLARE_SYSCTL; + + DECLARE_HYPERCALL_BOUNCE(d_info, max_domains*sizeof(*d_info), XC_HYPERCALL_BUFFER_BOUNCE_OUT); + + if ( xc_hypercall_bounce_pre(xch, d_info) ) + return -1; + + sysctl.cmd = XEN_SYSCTL_getdomcqminfolist; + sysctl.u.getdomcqminfolist.first_domain = first_domain; + sysctl.u.getdomcqminfolist.max_domains = max_domains; + set_xen_guest_handle(sysctl.u.getdomcqminfolist.buffer, d_info); + + if ( xc_sysctl(xch, &sysctl) < 0 ) + ret = -1; + else + { + ret = sysctl.u.getdomcqminfolist.num_domains; + info->resource_count = sysctl.u.getdomcqminfolist.resource_count; + info->resource_avail = sysctl.u.getdomcqminfolist.resource_avail; + } + + xc_hypercall_bounce_post(xch, d_info); + + return ret; +} + /* * Local variables: * mode: C diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h index a57f147..b0b1259 100644 --- a/tools/libxc/xenctrl.h +++ b/tools/libxc/xenctrl.h @@ -2395,6 +2395,11 @@ int xc_kexec_load(xc_interface *xch, uint8_t type, uint16_t arch, */ int xc_kexec_unload(xc_interface *xch, int type); +typedef xen_sysctl_getcqminfo_t sysctl_cqminfo_t; int xc_domain_pqos_attach(xc_interface *xch, uint32_t domid, int flags); int xc_domain_pqos_detach(xc_interface *xch, uint32_t domid, int flags); +int xc_domain_getcqminfolist(xc_interface *xch, + uint32_t first_domain, + unsigned int max_domains, + sysctl_cqminfo_t *info); #endif /* XENCTRL_H */ diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index a9a506f..530363c 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -285,6 +285,7 @@ #include <libxl_uuid.h> #include <_libxl_list.h> +#include <xenctrl.h> /* API compatibility. */ #ifdef LIBXL_API_VERSION @@ -1053,6 +1054,10 @@ int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size); int libxl_pqos_attach(libxl_ctx *ctx, uint32_t domid, const char * qosres); int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres); +int libxl_list_domain_cqm(libxl_ctx *ctx, + uint32_t first_domain, + unsigned int max_domains, + sysctl_cqminfo_t *info); /* misc */ diff --git a/tools/libxl/libxl_pqos.c b/tools/libxl/libxl_pqos.c index 6d8013a..e43c706 100644 --- a/tools/libxl/libxl_pqos.c +++ b/tools/libxl/libxl_pqos.c @@ -77,6 +77,20 @@ int libxl_pqos_detach(libxl_ctx *ctx, uint32_t domid, const char * qosres) return rc; } +int libxl_list_domain_cqm(libxl_ctx *ctx, + uint32_t first_domain, + unsigned int max_domains, + sysctl_cqminfo_t *info) +{ + int ret; + + ret = xc_domain_getcqminfolist(ctx->xch, first_domain, max_domains, info); + if (ret < 0) + return -EINVAL; + + return ret; +} + /* * Local variables: * mode: C diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h index 78738b9..994d3be 100644 --- a/tools/libxl/xl.h +++ b/tools/libxl/xl.h @@ -107,6 +107,7 @@ int main_loadpolicy(int argc, char **argv); int main_remus(int argc, char **argv); int main_pqosattach(int argc, char **argv); int main_pqosdetach(int argc, char **argv); +int main_pqoslist(int argc, char **argv); void help(const char *command); diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 1a12e8f..52c1321 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -7231,6 +7231,97 @@ int main_pqosdetach(int argc, char **argv) return rc; } +static void print_cqm_info(const sysctl_cqminfo_t *info, int nb_domain) +{ + int i, j; + int socket_num = 0; + xen_domctl_getdomcqminfo_t *dom_cqminfo; + + printf("RMID count %5d\tRMID available %5d\n", info->resource_count, info->resource_avail); + if (info->resource_count == 0) + printf("System doesn''t supoort CQM\n"); + else if (info->resource_count - info->resource_avail == 1) + printf("No RMID is assigned to domains\n"); + else if (!nb_domain) + printf("No RMID is assgined to your monitoring domains\n"); + else { + printf("Name ID"); + do { + printf("\tSocketID\tL3C_Usage"); + socket_num++; + } while (info->dom_cqminfo->socket_cqmdata[socket_num].valid); + printf("\n"); + for (i = 0; i < nb_domain; i++) { + char *domname; + dom_cqminfo = info->dom_cqminfo + i; + domname = libxl_domid_to_name(ctx, dom_cqminfo->domain); + printf("%-40s %5d", + domname, + dom_cqminfo->domain); + free(domname); + for(j = 0; j < socket_num; j++) + printf("%10u %16lu ", j, dom_cqminfo->socket_cqmdata[j].l3c_occupancy); + printf("\n"); + } + } +} +int main_pqoslist(int argc, char **argv) +{ + int opt; + const char *qosres = NULL; + uint32_t first_domain; + unsigned int max_domains; + int num_domains; + sysctl_cqminfo_t info; + + SWITCH_FOREACH_OPT(opt, "", NULL, "pqos-list", 1) { + /* No options */ + } + + qosres = argv[optind]; + + if (!strcmp(qosres, "cqm")) { + if (optind + 1 >= argc) { + first_domain = 0; + max_domains = 1024; + } else if (optind + 1 == argc - 1) { + first_domain = find_domain(argv[optind + 1]); + max_domains = 1; + } else { + help("pqos-list"); + return 2; + } + + info.dom_cqminfo = calloc(max_domains, + sizeof(xen_domctl_getdomcqminfo_t)); + if (!info.dom_cqminfo) { + fprintf(stderr, "allocating domain cqminfo failed.\n"); + return ERROR_FAIL; + } + + num_domains = libxl_list_domain_cqm(ctx, first_domain, + max_domains, &info); + + if (num_domains < 0) { + fprintf(stderr, "Failed to get domain CQM info, " + "check whether CQM feature is supported.\n"); + if (info.dom_cqminfo) + free(info.dom_cqminfo); + return 1; + } + print_cqm_info(&info, num_domains); + + if (info.dom_cqminfo) + free(info.dom_cqminfo); + } else { + fprintf(stderr, "qos resource type supported is: cqm\n"); + help("pqos-list"); + return 2; + } + + return 0; +} + /* * Local variables: * mode: C diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index 02a2572..6ced416 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -498,6 +498,11 @@ struct cmd_spec cmd_table[] = { "Reliquish qos resource", "<Resource> <Domain>", }, + { "pqos-list", + &main_pqoslist, 0, 0, + "List qos information about all/some domains", + "<Resource> [Domain]", + }, }; int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec); -- 1.7.9.5
dongxiao.xu@intel.com
2013-Nov-20 03:27 UTC
[PATCH 8/8] x86: enable CQM monitoring for each domain RMID
From: Dongxiao Xu <dongxiao.xu@intel.com> If the CQM service is attached to a domain, its related RMID will be set to hardware for monitoring when the domain''s vcpu is scheduled in. When the domain''s vcpu is scheduled out, RMID 0 (system reserved) will be set for monitoring. Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> --- xen/arch/x86/domain.c | 5 +++++ xen/arch/x86/pqos.c | 21 ++++++++++++++++++++- xen/include/asm-x86/msr-index.h | 1 + xen/include/asm-x86/pqos.h | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 5f920cd..77d3b12 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -1375,6 +1375,8 @@ static void __context_switch(void) { memcpy(&p->arch.user_regs, stack_regs, CTXT_SWITCH_STACK_BYTES); vcpu_save_fpu(p); + if ( system_support_cqm() ) + cqm_assoc_rmid(0); p->arch.ctxt_switch_from(p); } @@ -1399,6 +1401,9 @@ static void __context_switch(void) } vcpu_restore_fpu_eager(n); n->arch.ctxt_switch_to(n); + + if ( system_support_cqm() && n->domain->arch.pqos_cqm_rmid > 0 ) + cqm_assoc_rmid(n->domain->arch.pqos_cqm_rmid); } gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) : diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c index 3699efe..4b01a7a 100644 --- a/xen/arch/x86/pqos.c +++ b/xen/arch/x86/pqos.c @@ -27,6 +27,7 @@ #include <asm/pqos.h> static bool_t pqos_enabled = 1; +static unsigned int rmid_bitwidth; boolean_param("pqos", pqos_enabled); static void read_qm_data(void *arg) @@ -43,6 +44,14 @@ static void get_generic_qm_info(struct qm_element *qm_element) on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1); } +void cqm_assoc_rmid(uint16_t rmid) +{ + uint64_t val; + uint64_t rmid_mask = ~(~0ull << rmid_bitwidth); + rdmsrl(MSR_IA32_PQR_ASSOC, val); + wrmsrl(MSR_IA32_PQR_ASSOC, (val & ~(rmid_mask)) | (rmid & rmid_mask)); +} + unsigned int cqm_res_count = 0; unsigned int cqm_upscaling_factor = 0; bool_t cqm_enabled = 0; @@ -78,13 +87,23 @@ static void init_cqm(void) static void init_qos_monitor(void) { unsigned int qm_features; - unsigned int eax, ebx, ecx; + unsigned int eax, ebx, ecx, i = 0; if ( !(boot_cpu_has(X86_FEATURE_QOSM)) ) return; cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features); + while (1) + { + if ( (1 << i) - 1 >= ebx ) + { + rmid_bitwidth = i; + break; + } + i++; + } + if ( qm_features & QOS_MONITOR_TYPE_L3 ) init_cqm(); } diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h index 46ef165..45f4918 100644 --- a/xen/include/asm-x86/msr-index.h +++ b/xen/include/asm-x86/msr-index.h @@ -491,5 +491,6 @@ /* Platform QoS register */ #define MSR_IA32_QOSEVTSEL 0x00000c8d #define MSR_IA32_QMC 0x00000c8e +#define MSR_IA32_PQR_ASSOC 0x00000c8f #endif /* __ASM_MSR_INDEX_H */ diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h index 6d1b1e8..d5c5e37 100644 --- a/xen/include/asm-x86/pqos.h +++ b/xen/include/asm-x86/pqos.h @@ -51,5 +51,6 @@ unsigned int get_cqm_count(void); unsigned int get_cqm_avail(void); void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, struct xen_domctl_getdomcqminfo *info); +void cqm_assoc_rmid(uint16_t rmid); #endif -- 1.7.9.5
Andrew Cooper
2013-Nov-20 11:11 UTC
Re: [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:> From: Dongxiao Xu <dongxiao.xu@intel.com> > > Detect platform QoS feature status and enumerate the resource types, > one of which is to monitor the L3 cache occupancy. > > Also introduce a Xen grub command line parameter to control the > QoS feature status globally. > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > --- > xen/arch/x86/Makefile | 1 + > xen/arch/x86/cpu/intel.c | 6 +++ > xen/arch/x86/pqos.c | 89 ++++++++++++++++++++++++++++++++++++++ > xen/arch/x86/setup.c | 3 ++ > xen/include/asm-x86/cpufeature.h | 1 + > xen/include/asm-x86/pqos.h | 37 ++++++++++++++++ > 6 files changed, 137 insertions(+) > create mode 100644 xen/arch/x86/pqos.c > create mode 100644 xen/include/asm-x86/pqos.h > > diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile > index d502bdf..54962e0 100644 > --- a/xen/arch/x86/Makefile > +++ b/xen/arch/x86/Makefile > @@ -58,6 +58,7 @@ obj-y += crash.o > obj-y += tboot.o > obj-y += hpet.o > obj-y += xstate.o > +obj-y += pqos.o > > obj-$(crash_debug) += gdbstub.o > > diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c > index 27fe762..f0d83ea 100644 > --- a/xen/arch/x86/cpu/intel.c > +++ b/xen/arch/x86/cpu/intel.c > @@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c) > ( c->cpuid_level >= 0x00000006 ) && > ( cpuid_eax(0x00000006) & (1u<<2) ) ) > set_bit(X86_FEATURE_ARAT, c->x86_capability); > + > + /* Check platform QoS monitoring capability */ > + if ((c->cpuid_level >= 0x00000007) && > + (cpuid_ebx(0x00000007) & (1u<<12))) > + set_bit(X86_FEATURE_QOSM, c->x86_capability); > + > } > > static struct cpu_dev intel_cpu_dev __cpuinitdata = { > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > new file mode 100644 > index 0000000..617cce1 > --- /dev/null > +++ b/xen/arch/x86/pqos.c > @@ -0,0 +1,89 @@ > +/* > + * pqos.c: Platform QoS related service for guest. > + * > + * Copyright (c) 2013, Intel Corporation > + * Author: Jiongxi Li <jiongxi.li@intel.com> > + * Author: Dongxiao Xu <dongxiao.xu@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License along with > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple > + * Place - Suite 330, Boston, MA 02111-1307 USA. > + */ > +#include <asm/processor.h> > +#include <xen/init.h> > +#include <asm/pqos.h> > + > +static bool_t pqos_enabled = 1; > +boolean_param("pqos", pqos_enabled); > + > +unsigned int cqm_res_count = 0; > +unsigned int cqm_upscaling_factor = 0; > +bool_t cqm_enabled = 0; > +struct cqm_res_struct *cqm_res_array = NULL; > + > +static void init_cqm(void)__init> +{ > + unsigned int eax, edx; > + unsigned int max_cqm_rmid; > + > + cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid, &edx); > + if ( !(edx & QOS_MONITOR_EVTID_L3) ) > + return; > + > + cqm_res_count = max_cqm_rmid + 1; > + > + cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count); > + if ( !cqm_res_array ) > + { > + cqm_res_count = 0; > + return; > + } > + > + cqm_enabled = 1; > + > + /* Reserve RMID 0 for all domains not being monitored */ > + cqm_res_array[0].inuse = 1; > + > + printk("Cache QoS Monitoring Enabled.\n");This should probably be a XENLOG_INFO, and we tend to omit full stops.> +} > + > +static void init_qos_monitor(void)__init> +{ > + unsigned int qm_features; > + unsigned int eax, ebx, ecx; > + > + if ( !(boot_cpu_has(X86_FEATURE_QOSM)) ) > + return; > + > + cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features); > + > + if ( qm_features & QOS_MONITOR_TYPE_L3 ) > + init_cqm(); > +} > + > +void init_platform_qos(void)__init Are there expected to be future platform qos monitoring abilities?> +{ > + if ( !pqos_enabled ) > + return; > + > + init_qos_monitor(); > +} > + > +/* > + * Local variables: > + * mode: C > + * c-file-style: "BSD" > + * c-basic-offset: 4 > + * tab-width: 4 > + * indent-tabs-mode: nil > + * End: > + */ > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c > index 5bf4ee0..95418e4 100644 > --- a/xen/arch/x86/setup.c > +++ b/xen/arch/x86/setup.c > @@ -48,6 +48,7 @@ > #include <asm/setup.h> > #include <xen/cpu.h> > #include <asm/nmi.h> > +#include <asm/pqos.h> > > /* opt_nosmp: If true, secondary processors are ignored. */ > static bool_t __initdata opt_nosmp; > @@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p) > > domain_unpause_by_systemcontroller(dom0); > > + init_platform_qos(); > + > reset_stack_and_jump(init_done); > } > > diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h > index 1cfaf94..ca59668 100644 > --- a/xen/include/asm-x86/cpufeature.h > +++ b/xen/include/asm-x86/cpufeature.h > @@ -147,6 +147,7 @@ > #define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */ > #define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context ID */ > #define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional Memory */ > +#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring capability */ > #define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */ > #define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access Prevention */ > > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > new file mode 100644 > index 0000000..2184ea9 > --- /dev/null > +++ b/xen/include/asm-x86/pqos.h > @@ -0,0 +1,37 @@ > +/* > + * pqos.h: Platform QoS related service for guest. > + * > + * Copyright (c) 2013, Intel Corporation > + * Author: Jiongxi Li <jiongxi.li@intel.com> > + * Author: Dongxiao Xu <dongxiao.xu@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License along with > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple > + * Place - Suite 330, Boston, MA 02111-1307 USA. > + */ > +#ifndef ASM_PQOS_H > +#define ASM_PQOS_H > + > +/* QoS Resource Type Enumeration */ > +#define QOS_MONITOR_TYPE_L3 0x2 > + > +/* QoS Monitoring Event ID */ > +#define QOS_MONITOR_EVTID_L3 0x1 > + > +struct cqm_res_struct { > + bool_t inuse; > + uint16_t domain_id;domid_t please. Also, reversing the order of the parameters would be a good idea, packing-wise. ~Andrew> +}; > + > +void init_platform_qos(void); > + > +#endif
Andrew Cooper
2013-Nov-20 11:32 UTC
Re: [PATCH 2/8] x86: handle CQM resource when creating/destroying guests
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:> From: Dongxiao Xu <dongxiao.xu@intel.com> > > Allocate an RMID for a guest when it is created. This per-guest > RMID will be used to monitor Cache QoS related data. The RMID will > be relinquished when guest is destroyed. > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > --- > xen/arch/x86/domain.c | 12 +++++++++ > xen/arch/x86/pqos.c | 58 ++++++++++++++++++++++++++++++++++++++++++ > xen/common/domctl.c | 5 +++- > xen/include/asm-x86/domain.h | 2 ++ > xen/include/asm-x86/pqos.h | 4 +++ > xen/include/public/domctl.h | 3 +++ > xen/include/xen/sched.h | 3 +++ > 7 files changed, 86 insertions(+), 1 deletion(-) > > diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c > index a3868f9..5f920cd 100644 > --- a/xen/arch/x86/domain.c > +++ b/xen/arch/x86/domain.c > @@ -60,6 +60,7 @@ > #include <xen/numa.h> > #include <xen/iommu.h> > #include <compat/vcpu.h> > +#include <asm/pqos.h> > > DEFINE_PER_CPU(struct vcpu *, curr_vcpu); > DEFINE_PER_CPU(unsigned long, cr4); > @@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags) > tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0); > spin_lock_init(&d->arch.vtsc_lock); > > + /* Allocate CQM RMID for guest */ > + if ( !!(domcr_flags & DOMCRF_pqos_cqm) ) > + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id); > + else if ( system_support_cqm() ) > + d->arch.pqos_cqm_rmid = 0; > + else > + d->arch.pqos_cqm_rmid = -1; > +This logic looks overly-complicated, and calling alloc_cqm_resource() before checking system_support_cqm() seems the wrong way around.> return 0; > > fail: > @@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d) > > free_xenheap_page(d->shared_info); > cleanup_domain_irq_mapping(d); > + > + free_cqm_resource(d->domain_id); > + d->arch.pqos_cqm_rmid = 0; > } > > unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long guest_cr4) > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > index 617cce1..895d892 100644 > --- a/xen/arch/x86/pqos.c > +++ b/xen/arch/x86/pqos.c > @@ -20,6 +20,7 @@ > */ > #include <asm/processor.h> > #include <xen/init.h> > +#include <xen/spinlock.h> > #include <asm/pqos.h> > > static bool_t pqos_enabled = 1; > @@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0; > unsigned int cqm_upscaling_factor = 0; > bool_t cqm_enabled = 0; > struct cqm_res_struct *cqm_res_array = NULL; > +static DEFINE_SPINLOCK(cqm_lock); > > static void init_cqm(void) > { > @@ -78,6 +80,62 @@ void init_platform_qos(void) > init_qos_monitor(); > } > > +bool_t system_support_cqm(void)"system_supports_cqm"> +{ > + return cqm_enabled; > +} > + > +int alloc_cqm_resource(domid_t domain_id)Perhaps alloc_rmid() given its current semantics?> +{ > + int i, rmid = -1; > + unsigned long flags; > + > + if ( !system_support_cqm() ) > + goto out;Probably better as an ASSERT(). The caller should really not try to allocate cqm resources if the system doesn''t have cqm enabled.> + > + spin_lock_irqsave(&cqm_lock, flags); > + for ( i = 0; i < cqm_res_count; i++ ) > + { > + if ( cqm_res_array[i].inuse) > + continue; > + > + rmid = i; > + cqm_res_array[rmid].inuse = 1; > + cqm_res_array[rmid].domain_id = domain_id; > + break; > + } > + spin_unlock_irqrestore(&cqm_lock, flags); > + > + /* No CQM RMID available, assign RMID=0 by default */ > + if ( rmid == -1 ) > + rmid = 0; > + > +out: > + return rmid; > +} > + > +void free_cqm_resource(domid_t domain_id) > +{ > + int rmid; > + unsigned long flags; > + > + if ( !system_support_cqm() ) > + return; > + > + spin_lock_irqsave(&cqm_lock, flags); > + /* CQM RMID 0 is reserved, enumerate from 1 */ > + for ( rmid = 1; rmid < cqm_res_count; rmid++ ) > + { > + if ( cqm_res_array[rmid].domain_id != domain_id ) > + continue; > + > + cqm_res_array[rmid].inuse = 0; > + cqm_res_array[rmid].domain_id = 0; > + break; > + } > + spin_unlock_irqrestore(&cqm_lock, flags);Can you not use d->arch.pqos_cqm_rmid here to save the search ?> +} > + > /* > * Local variables: > * mode: C > diff --git a/xen/common/domctl.c b/xen/common/domctl.c > index 904d27b..1c2e320 100644 > --- a/xen/common/domctl.c > +++ b/xen/common/domctl.c > @@ -425,7 +425,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) > | XEN_DOMCTL_CDF_pvh_guest > | XEN_DOMCTL_CDF_hap > | XEN_DOMCTL_CDF_s3_integrity > - | XEN_DOMCTL_CDF_oos_off)) ) > + | XEN_DOMCTL_CDF_oos_off > + | XEN_DOMCTL_CDF_pqos_cqm)) ) > break; > > dom = op->domain; > @@ -467,6 +468,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) > domcr_flags |= DOMCRF_s3_integrity; > if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off ) > domcr_flags |= DOMCRF_oos_off; > + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm ) > + domcr_flags |= DOMCRF_pqos_cqm; > > d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref); > if ( IS_ERR(d) ) > diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h > index 9d39061..b0479aa 100644 > --- a/xen/include/asm-x86/domain.h > +++ b/xen/include/asm-x86/domain.h > @@ -313,6 +313,8 @@ struct arch_domain > spinlock_t e820_lock; > struct e820entry *e820; > unsigned int nr_e820; > + > + int pqos_cqm_rmid; /* CQM RMID assigned to the domain */ > } __cacheline_aligned; > > #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list)) > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > index 2184ea9..7e32fa5 100644 > --- a/xen/include/asm-x86/pqos.h > +++ b/xen/include/asm-x86/pqos.h > @@ -34,4 +34,8 @@ struct cqm_res_struct { > > void init_platform_qos(void); > > +bool_t system_support_cqm(void); > +int alloc_cqm_resource(domid_t); > +void free_cqm_resource(domid_t); > + > #endif > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > index 01a3652..47a850a 100644 > --- a/xen/include/public/domctl.h > +++ b/xen/include/public/domctl.h > @@ -62,6 +62,9 @@ struct xen_domctl_createdomain { > /* Is this a PVH guest (as opposed to an HVM or PV guest)? */ > #define _XEN_DOMCTL_CDF_pvh_guest 4 > #define XEN_DOMCTL_CDF_pvh_guest (1U<<_XEN_DOMCTL_CDF_pvh_guest) > + /* Enable pqos-cqm? */ > +#define _XEN_DOMCTL_CDF_pqos_cqm 5 > +#define XEN_DOMCTL_CDF_pqos_cqm (1U<<_XEN_DOMCTL_CDF_pqos_cqm) > uint32_t flags; > }; > typedef struct xen_domctl_createdomain xen_domctl_createdomain_t; > diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h > index cbdf377..3a42656 100644 > --- a/xen/include/xen/sched.h > +++ b/xen/include/xen/sched.h > @@ -507,6 +507,9 @@ struct domain *domain_create( > /* DOMCRF_pvh: Create PV domain in HVM container. */ > #define _DOMCRF_pvh 5 > #define DOMCRF_pvh (1U<<_DOMCRF_pvh) > + /* DOMCRF_pqos_cqm: Create a domain with CQM support */ > +#define _DOMCRF_pqos_cqm 6 > +#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm) > > /* > * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
Andrew Cooper
2013-Nov-20 11:44 UTC
Re: [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:> From: Dongxiao Xu <dongxiao.xu@intel.com> > > Add hypervisor side support for dynamically attach and detach CQM > services for a certain guest. > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > --- > xen/arch/x86/domctl.c | 66 +++++++++++++++++++++++++++++++++++++++++++ > xen/include/public/domctl.h | 15 ++++++++++ > 2 files changed, 81 insertions(+) > > diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c > index f7e4586..df6a373 100644 > --- a/xen/arch/x86/domctl.c > +++ b/xen/arch/x86/domctl.c > @@ -35,6 +35,7 @@ > #include <asm/mem_sharing.h> > #include <asm/xstate.h> > #include <asm/debugger.h> > +#include <asm/pqos.h> > > static int gdbsx_guest_mem_io( > domid_t domid, struct xen_domctl_gdbsx_memio *iop) > @@ -1223,6 +1224,71 @@ long arch_do_domctl( > } > break; > > + case XEN_DOMCTL_attach_pqos: > + { > + struct domain *d; > + > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) > + { > + ret = -EBUSY; > + break; > + }do_domctl has done this for us. There is also no need to shadow ''d''.> + > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) > + { > + if ( !system_support_cqm() ) > + ret = -ENODEV;Surely this is generic, rather than condition on the caller specifying XEN_DOMCTL_ADF_pqos_cqm> + else if ( d->arch.pqos_cqm_rmid > 0 ) > + ret = -EINVAL; > + else > + { > + ret = 0; > + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id); > + if ( d->arch.pqos_cqm_rmid <= 0 ) > + { > + /* set to default value */ > + d->arch.pqos_cqm_rmid = 0; > + ret = -ENODEV;Perhaps E2BIG or EBUSY ? This error can be fixed by detaching cqm from other domains.> + } > + } > + } > + else > + ret = -EINVAL; > + > + rcu_unlock_domain(d); > + } > + break; > + > + case XEN_DOMCTL_detach_pqos: > + { > + struct domain *d; > + > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) > + { > + ret = -EBUSY; > + break; > + } > + > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) > + { > + if ( !system_support_cqm() ) > + ret = -ENODEV; > + else if ( d->arch.pqos_cqm_rmid > 0 ) > + { > + free_cqm_resource(d->domain_id); > + d->arch.pqos_cqm_rmid = 0; > + ret = 0; > + } > + else > + ret = -EINVAL; > + } > + else > + ret = -EINVAL; > + > + rcu_unlock_domain(d); > + } > + break; > + > default: > ret = iommu_do_domctl(domctl, d, u_domctl); > break; > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > index 47a850a..4fe21db 100644 > --- a/xen/include/public/domctl.h > +++ b/xen/include/public/domctl.h > @@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn { > typedef struct xen_domctl_set_max_evtchn xen_domctl_set_max_evtchn_t; > DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t); > > +/* XEN_DOMCTL_attach_pqos */ > +/* XEN_DOMCTL_detach_pqos */ > +struct xen_domctl_qos_resource { > + /* Attach or detach flag for cqm */ > +#define _XEN_DOMCTL_ADF_pqos_cqm 0 > +#define XEN_DOMCTL_ADF_pqos_cqm (1U<<_XEN_DOMCTL_ADF_pqos_cqm) > + uint32_t flags;The implementation semantics of this flag is redundant. You differentiate attach/detach both on the domctl subop, and the flag in this structure, leaving two invalid states. It might be better to have a XEN_DOMCTL_qpos_op and a sub command, if more of these hypercalls are on their way.> +}; > +typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); > + > struct xen_domctl { > uint32_t cmd; > #define XEN_DOMCTL_createdomain 1 > @@ -941,6 +952,9 @@ struct xen_domctl { > #define XEN_DOMCTL_setnodeaffinity 68 > #define XEN_DOMCTL_getnodeaffinity 69 > #define XEN_DOMCTL_set_max_evtchn 70 > +#define XEN_DOMCTL_attach_pqos 71 > +#define XEN_DOMCTL_detach_pqos 72 > +#define XEN_DOMCTL_list_pqos 73You introduce this list_pqos domctl with no implementation.> #define XEN_DOMCTL_gdbsx_guestmemio 1000 > #define XEN_DOMCTL_gdbsx_pausevcpu 1001 > #define XEN_DOMCTL_gdbsx_unpausevcpu 1002 > @@ -1001,6 +1015,7 @@ struct xen_domctl { > struct xen_domctl_set_broken_page_p2m set_broken_page_p2m; > struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu; > struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; > + struct xen_domctl_qos_resource qos_res; > uint8_t pad[128]; > } u; > };
On 20/11/13 03:27, dongxiao.xu@intel.com wrote:> From: Dongxiao Xu <dongxiao.xu@intel.com> > > Retrive CQM information for certain domain, which reflects the L3 cache > occupancy for a socket. > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > --- > xen/arch/x86/pqos.c | 60 ++++++++++++++++++++++++++++++++++++ > xen/arch/x86/sysctl.c | 64 +++++++++++++++++++++++++++++++++++++++ > xen/include/asm-x86/msr-index.h | 4 +++ > xen/include/asm-x86/pqos.h | 14 +++++++++ > xen/include/public/domctl.h | 14 +++++++++ > xen/include/public/sysctl.h | 23 ++++++++++++++ > 6 files changed, 179 insertions(+) > > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > index 895d892..3699efe 100644 > --- a/xen/arch/x86/pqos.c > +++ b/xen/arch/x86/pqos.c > @@ -19,13 +19,30 @@ > * Place - Suite 330, Boston, MA 02111-1307 USA. > */ > #include <asm/processor.h> > +#include <asm/msr.h> > +#include <xen/cpumask.h> > #include <xen/init.h> > #include <xen/spinlock.h> > +#include <public/domctl.h> > #include <asm/pqos.h> > > static bool_t pqos_enabled = 1; > boolean_param("pqos", pqos_enabled); > > +static void read_qm_data(void *arg) > +{ > + struct qm_element *qm_element = arg; > + > + wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid, qm_element->rmid); > + rdmsrl(MSR_IA32_QMC, qm_element->qm_data); > +} > + > +static void get_generic_qm_info(struct qm_element *qm_element) > +{ > + int cpu = qm_element->cpu; > + on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1); > +} > + > unsigned int cqm_res_count = 0; > unsigned int cqm_upscaling_factor = 0; > bool_t cqm_enabled = 0; > @@ -85,6 +102,25 @@ bool_t system_support_cqm(void) > return cqm_enabled; > } > > +unsigned int get_cqm_count(void) > +{ > + return cqm_res_count; > +} > + > +unsigned int get_cqm_avail(void) > +{ > + unsigned int cqm_avail = 0; > + int i; > + > + for (i = 0; i < cqm_res_count; i++) > + { > + if ( !cqm_res_array[i].inuse ) > + cqm_avail++; > + }Style - extra spaces inside brackets for the for loop, and these braces can go.> + > + return cqm_avail; > +} > + > int alloc_cqm_resource(domid_t domain_id) > { > int i, rmid = -1; > @@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id) > spin_unlock_irqrestore(&cqm_lock, flags); > } > > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, > + struct xen_domctl_getdomcqminfo *info) > +{ > + struct qm_element element; > + int cpu, i; > + > + for_each_cpu ( cpu, &cpu_cqmdata_map ) > + { > + element.cpu = cpu; > + element.rmid = rmid; > + element.evtid = QOS_MONITOR_EVTID_L3; > + > + get_generic_qm_info(&element); > + > + i = cpu_to_socket(cpu); > + info->socket_cqmdata[i].valid > + (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1; > + if ( info->socket_cqmdata[i].valid ) > + info->socket_cqmdata[i].l3c_occupancy = element.qm_data * cqm_upscaling_factor; > + else > + info->socket_cqmdata[i].l3c_occupancy = 0; > + } > +} > + > /* > * Local variables: > * mode: C > diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c > index 15d4b91..a779fdc 100644 > --- a/xen/arch/x86/sysctl.c > +++ b/xen/arch/x86/sysctl.c > @@ -28,6 +28,7 @@ > #include <xen/nodemask.h> > #include <xen/cpu.h> > #include <xsm/xsm.h> > +#include <asm/pqos.h> > > #define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0) > > @@ -101,6 +102,69 @@ long arch_do_sysctl( > } > break; > > + case XEN_SYSCTL_getdomcqminfolist: > + { > + struct domain *d; > + struct xen_domctl_getdomcqminfo info; > + uint32_t resource_count; > + uint32_t resource_avail; > + uint32_t num_domains = 0; > + cpumask_t cpu_cqmdata_map; > + DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS); > + int cpu;unsigned int.> + > + if ( !system_support_cqm() ) > + { > + ret = -EFAULT;ENODEV surely ?> + break; > + } > + > + resource_count = get_cqm_count(); > + resource_avail = get_cqm_avail(); > + > + cpumask_clear(&cpu_cqmdata_map); > + bitmap_zero(sockets, QOS_MAX_SOCKETS); > + for_each_online_cpu(cpu) > + { > + int i = cpu_to_socket(cpu); > + if ( test_and_set_bit(i, sockets) ) > + continue; > + cpumask_set_cpu(cpu, &cpu_cqmdata_map); > + } > + > + rcu_read_lock(&domlist_read_lock); > + for_each_domain ( d ) > + { > + if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain ) > + continue; > + if ( num_domains == sysctl->u.getdomaininfolist.max_domains ) > + break; > + if ( d->arch.pqos_cqm_rmid <= 0 ) > + continue; > + memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo)); > + info.domain = d->domain_id; > + get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map, &info); > + > + if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer, > + num_domains, &info, 1) ) > + { > + ret = -EFAULT; > + break; > + } > + > + num_domains++; > + } > + rcu_read_unlock(&domlist_read_lock); > + > + sysctl->u.getdomcqminfolist.num_domains = num_domains; > + sysctl->u.getdomcqminfolist.resource_count = resource_count; > + sysctl->u.getdomcqminfolist.resource_avail = resource_avail; > + > + if ( copy_to_guest(u_sysctl, sysctl, 1) ) > + ret = -EFAULT; > + } > + break; > + > default: > ret = -ENOSYS; > break; > diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h > index e597a28..46ef165 100644 > --- a/xen/include/asm-x86/msr-index.h > +++ b/xen/include/asm-x86/msr-index.h > @@ -488,4 +488,8 @@ > /* Geode defined MSRs */ > #define MSR_GEODE_BUSCONT_CONF0 0x00001900 > > +/* Platform QoS register */ > +#define MSR_IA32_QOSEVTSEL 0x00000c8d > +#define MSR_IA32_QMC 0x00000c8e > + > #endif /* __ASM_MSR_INDEX_H */ > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > index 7e32fa5..6d1b1e8 100644 > --- a/xen/include/asm-x86/pqos.h > +++ b/xen/include/asm-x86/pqos.h > @@ -27,15 +27,29 @@ > /* QoS Monitoring Event ID */ > #define QOS_MONITOR_EVTID_L3 0x1 > > +/* IA32_QM_CTR */ > +#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62) > + > struct cqm_res_struct { > bool_t inuse; > uint16_t domain_id; > }; > > +struct qm_element { > + uint16_t cpu; > + uint32_t rmid; > + uint8_t evtid; > + uint64_t qm_data; > +}; > +The packing of this structure is rather poor. Please re-order to reduce some of the holes. Also, cpu parameters are unsigned int in Xen.> void init_platform_qos(void); > > bool_t system_support_cqm(void); > int alloc_cqm_resource(domid_t); > void free_cqm_resource(domid_t); > +unsigned int get_cqm_count(void); > +unsigned int get_cqm_avail(void); > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, > + struct xen_domctl_getdomcqminfo *info); > > #endif > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > index 4fe21db..bdefc83 100644 > --- a/xen/include/public/domctl.h > +++ b/xen/include/public/domctl.h > @@ -883,6 +883,20 @@ struct xen_domctl_qos_resource { > typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; > DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); > > +struct xen_socket_cqmdata { > + uint8_t valid; > + uint64_t l3c_occupancy; > +}; > + > +struct xen_domctl_getdomcqminfo { > + /* OUT variables. */ > + domid_t domain; > +#define QOS_MAX_SOCKETS 128 > + struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS]; > +}; > +typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t; > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t); > + > struct xen_domctl { > uint32_t cmd; > #define XEN_DOMCTL_createdomain 1 > diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h > index 8437d31..91f206e 100644 > --- a/xen/include/public/sysctl.h > +++ b/xen/include/public/sysctl.h > @@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op { > typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t; > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t); > > +struct xen_sysctl_getcqminfo > +{ > + uint32_t resource_count; > + uint32_t resource_avail; > + struct xen_domctl_getdomcqminfo *dom_cqminfo;XEN_GUEST_HANDLE please.> +}; > +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t; > + > /* XEN_SYSCTL_getdomaininfolist */ > struct xen_sysctl_getdomaininfolist { > /* IN variables. */ > @@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op { > typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t; > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t); > > +/* XEN_SYSCTL_getdomcqminfolist */ > +struct xen_sysctl_getdomcqminfolist { > + /* IN variables. */ > + domid_t first_domain; > + uint32_t max_domains; > + XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer; > + /* OUT variables. */ > + uint32_t num_domains; > + uint32_t resource_count; > + uint32_t resource_avail; > +}; > +typedef struct xen_sysctl_getdomcqminfolist xen_sysctl_getdomcqminfolist_t; > +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t); > > struct xen_sysctl { > uint32_t cmd; > @@ -654,6 +675,7 @@ struct xen_sysctl { > #define XEN_SYSCTL_cpupool_op 18 > #define XEN_SYSCTL_scheduler_op 19 > #define XEN_SYSCTL_coverage_op 20 > +#define XEN_SYSCTL_getdomcqminfolist 21 > uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */ > union { > struct xen_sysctl_readconsole readconsole; > @@ -675,6 +697,7 @@ struct xen_sysctl { > struct xen_sysctl_cpupool_op cpupool_op; > struct xen_sysctl_scheduler_op scheduler_op; > struct xen_sysctl_coverage_op coverage_op; > + struct xen_sysctl_getdomcqminfolist getdomcqminfolist; > uint8_t pad[128]; > } u; > };
Xu, Dongxiao
2013-Nov-20 12:40 UTC
Re: [PATCH 1/8] x86: detect and initialize Cache QoS Monitoring feature
> -----Original Message----- > From: Andrew Cooper [mailto:andrew.cooper3@citrix.com] > Sent: Wednesday, November 20, 2013 7:12 PM > To: Xu, Dongxiao > Cc: xen-devel@lists.xen.org > Subject: Re: [Xen-devel] [PATCH 1/8] x86: detect and initialize Cache QoS > Monitoring feature > > On 20/11/13 03:27, dongxiao.xu@intel.com wrote: > > From: Dongxiao Xu <dongxiao.xu@intel.com> > > > > Detect platform QoS feature status and enumerate the resource types, > > one of which is to monitor the L3 cache occupancy. > > > > Also introduce a Xen grub command line parameter to control the > > QoS feature status globally. > > > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > > --- > > xen/arch/x86/Makefile | 1 + > > xen/arch/x86/cpu/intel.c | 6 +++ > > xen/arch/x86/pqos.c | 89 > ++++++++++++++++++++++++++++++++++++++ > > xen/arch/x86/setup.c | 3 ++ > > xen/include/asm-x86/cpufeature.h | 1 + > > xen/include/asm-x86/pqos.h | 37 ++++++++++++++++ > > 6 files changed, 137 insertions(+) > > create mode 100644 xen/arch/x86/pqos.c > > create mode 100644 xen/include/asm-x86/pqos.h > > > > diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile > > index d502bdf..54962e0 100644 > > --- a/xen/arch/x86/Makefile > > +++ b/xen/arch/x86/Makefile > > @@ -58,6 +58,7 @@ obj-y += crash.o > > obj-y += tboot.o > > obj-y += hpet.o > > obj-y += xstate.o > > +obj-y += pqos.o > > > > obj-$(crash_debug) += gdbstub.o > > > > diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c > > index 27fe762..f0d83ea 100644 > > --- a/xen/arch/x86/cpu/intel.c > > +++ b/xen/arch/x86/cpu/intel.c > > @@ -230,6 +230,12 @@ static void __devinit init_intel(struct cpuinfo_x86 *c) > > ( c->cpuid_level >= 0x00000006 ) && > > ( cpuid_eax(0x00000006) & (1u<<2) ) ) > > set_bit(X86_FEATURE_ARAT, c->x86_capability); > > + > > + /* Check platform QoS monitoring capability */ > > + if ((c->cpuid_level >= 0x00000007) && > > + (cpuid_ebx(0x00000007) & (1u<<12))) > > + set_bit(X86_FEATURE_QOSM, c->x86_capability); > > + > > } > > > > static struct cpu_dev intel_cpu_dev __cpuinitdata = { > > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > > new file mode 100644 > > index 0000000..617cce1 > > --- /dev/null > > +++ b/xen/arch/x86/pqos.c > > @@ -0,0 +1,89 @@ > > +/* > > + * pqos.c: Platform QoS related service for guest. > > + * > > + * Copyright (c) 2013, Intel Corporation > > + * Author: Jiongxi Li <jiongxi.li@intel.com> > > + * Author: Dongxiao Xu <dongxiao.xu@intel.com> > > + * > > + * This program is free software; you can redistribute it and/or modify it > > + * under the terms and conditions of the GNU General Public License, > > + * version 2, as published by the Free Software Foundation. > > + * > > + * This program is distributed in the hope it will be useful, but WITHOUT > > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > or > > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > License for > > + * more details. > > + * > > + * You should have received a copy of the GNU General Public License along > with > > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple > > + * Place - Suite 330, Boston, MA 02111-1307 USA. > > + */ > > +#include <asm/processor.h> > > +#include <xen/init.h> > > +#include <asm/pqos.h> > > + > > +static bool_t pqos_enabled = 1; > > +boolean_param("pqos", pqos_enabled); > > + > > +unsigned int cqm_res_count = 0; > > +unsigned int cqm_upscaling_factor = 0; > > +bool_t cqm_enabled = 0; > > +struct cqm_res_struct *cqm_res_array = NULL; > > + > > +static void init_cqm(void) > > __initOK.> > > +{ > > + unsigned int eax, edx; > > + unsigned int max_cqm_rmid; > > + > > + cpuid_count(0xf, 1, &eax, &cqm_upscaling_factor, &max_cqm_rmid, > &edx); > > + if ( !(edx & QOS_MONITOR_EVTID_L3) ) > > + return; > > + > > + cqm_res_count = max_cqm_rmid + 1; > > + > > + cqm_res_array = xzalloc_array(struct cqm_res_struct, cqm_res_count); > > + if ( !cqm_res_array ) > > + { > > + cqm_res_count = 0; > > + return; > > + } > > + > > + cqm_enabled = 1; > > + > > + /* Reserve RMID 0 for all domains not being monitored */ > > + cqm_res_array[0].inuse = 1; > > + > > + printk("Cache QoS Monitoring Enabled.\n"); > > This should probably be a XENLOG_INFO, and we tend to omit full stops.OK.> > > +} > > + > > +static void init_qos_monitor(void) > > __initOK.> > > +{ > > + unsigned int qm_features; > > + unsigned int eax, ebx, ecx; > > + > > + if ( !(boot_cpu_has(X86_FEATURE_QOSM)) ) > > + return; > > + > > + cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features); > > + > > + if ( qm_features & QOS_MONITOR_TYPE_L3 ) > > + init_cqm(); > > +} > > + > > +void init_platform_qos(void) > > __initOK.> > Are there expected to be future platform qos monitoring abilities?Yes.> > > +{ > > + if ( !pqos_enabled ) > > + return; > > + > > + init_qos_monitor(); > > +} > > + > > +/* > > + * Local variables: > > + * mode: C > > + * c-file-style: "BSD" > > + * c-basic-offset: 4 > > + * tab-width: 4 > > + * indent-tabs-mode: nil > > + * End: > > + */ > > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c > > index 5bf4ee0..95418e4 100644 > > --- a/xen/arch/x86/setup.c > > +++ b/xen/arch/x86/setup.c > > @@ -48,6 +48,7 @@ > > #include <asm/setup.h> > > #include <xen/cpu.h> > > #include <asm/nmi.h> > > +#include <asm/pqos.h> > > > > /* opt_nosmp: If true, secondary processors are ignored. */ > > static bool_t __initdata opt_nosmp; > > @@ -1402,6 +1403,8 @@ void __init __start_xen(unsigned long mbi_p) > > > > domain_unpause_by_systemcontroller(dom0); > > > > + init_platform_qos(); > > + > > reset_stack_and_jump(init_done); > > } > > > > diff --git a/xen/include/asm-x86/cpufeature.h > b/xen/include/asm-x86/cpufeature.h > > index 1cfaf94..ca59668 100644 > > --- a/xen/include/asm-x86/cpufeature.h > > +++ b/xen/include/asm-x86/cpufeature.h > > @@ -147,6 +147,7 @@ > > #define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */ > > #define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context > ID */ > > #define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional > Memory */ > > +#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring > capability */ > > #define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as > zero */ > > #define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access > Prevention */ > > > > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > > new file mode 100644 > > index 0000000..2184ea9 > > --- /dev/null > > +++ b/xen/include/asm-x86/pqos.h > > @@ -0,0 +1,37 @@ > > +/* > > + * pqos.h: Platform QoS related service for guest. > > + * > > + * Copyright (c) 2013, Intel Corporation > > + * Author: Jiongxi Li <jiongxi.li@intel.com> > > + * Author: Dongxiao Xu <dongxiao.xu@intel.com> > > + * > > + * This program is free software; you can redistribute it and/or modify it > > + * under the terms and conditions of the GNU General Public License, > > + * version 2, as published by the Free Software Foundation. > > + * > > + * This program is distributed in the hope it will be useful, but WITHOUT > > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY > or > > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > License for > > + * more details. > > + * > > + * You should have received a copy of the GNU General Public License along > with > > + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple > > + * Place - Suite 330, Boston, MA 02111-1307 USA. > > + */ > > +#ifndef ASM_PQOS_H > > +#define ASM_PQOS_H > > + > > +/* QoS Resource Type Enumeration */ > > +#define QOS_MONITOR_TYPE_L3 0x2 > > + > > +/* QoS Monitoring Event ID */ > > +#define QOS_MONITOR_EVTID_L3 0x1 > > + > > +struct cqm_res_struct { > > + bool_t inuse; > > + uint16_t domain_id; > > domid_t please. Also, reversing the order of the parameters would be a > good idea, packing-wise.OK. Will reflect above suggestions in next version patches. Thanks, Dongxiao> > ~Andrew > > > +}; > > + > > +void init_platform_qos(void); > > + > > +#endif
Xu, Dongxiao
2013-Nov-20 12:49 UTC
Re: [PATCH 2/8] x86: handle CQM resource when creating/destroying guests
> -----Original Message----- > From: Andrew Cooper [mailto:andrew.cooper3@citrix.com] > Sent: Wednesday, November 20, 2013 7:33 PM > To: Xu, Dongxiao > Cc: xen-devel@lists.xen.org > Subject: Re: [Xen-devel] [PATCH 2/8] x86: handle CQM resource when > creating/destroying guests > > On 20/11/13 03:27, dongxiao.xu@intel.com wrote: > > From: Dongxiao Xu <dongxiao.xu@intel.com> > > > > Allocate an RMID for a guest when it is created. This per-guest > > RMID will be used to monitor Cache QoS related data. The RMID will > > be relinquished when guest is destroyed. > > > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > > --- > > xen/arch/x86/domain.c | 12 +++++++++ > > xen/arch/x86/pqos.c | 58 > ++++++++++++++++++++++++++++++++++++++++++ > > xen/common/domctl.c | 5 +++- > > xen/include/asm-x86/domain.h | 2 ++ > > xen/include/asm-x86/pqos.h | 4 +++ > > xen/include/public/domctl.h | 3 +++ > > xen/include/xen/sched.h | 3 +++ > > 7 files changed, 86 insertions(+), 1 deletion(-) > > > > diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c > > index a3868f9..5f920cd 100644 > > --- a/xen/arch/x86/domain.c > > +++ b/xen/arch/x86/domain.c > > @@ -60,6 +60,7 @@ > > #include <xen/numa.h> > > #include <xen/iommu.h> > > #include <compat/vcpu.h> > > +#include <asm/pqos.h> > > > > DEFINE_PER_CPU(struct vcpu *, curr_vcpu); > > DEFINE_PER_CPU(unsigned long, cr4); > > @@ -579,6 +580,14 @@ int arch_domain_create(struct domain *d, unsigned int > domcr_flags) > > tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0); > > spin_lock_init(&d->arch.vtsc_lock); > > > > + /* Allocate CQM RMID for guest */ > > + if ( !!(domcr_flags & DOMCRF_pqos_cqm) ) > > + d->arch.pqos_cqm_rmid = alloc_cqm_resource(d->domain_id); > > + else if ( system_support_cqm() ) > > + d->arch.pqos_cqm_rmid = 0; > > + else > > + d->arch.pqos_cqm_rmid = -1; > > + > > This logic looks overly-complicated, and calling alloc_cqm_resource() > before checking system_support_cqm() seems the wrong way around.OK.> > > return 0; > > > > fail: > > @@ -612,6 +621,9 @@ void arch_domain_destroy(struct domain *d) > > > > free_xenheap_page(d->shared_info); > > cleanup_domain_irq_mapping(d); > > + > > + free_cqm_resource(d->domain_id); > > + d->arch.pqos_cqm_rmid = 0; > > } > > > > unsigned long pv_guest_cr4_fixup(const struct vcpu *v, unsigned long > guest_cr4) > > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > > index 617cce1..895d892 100644 > > --- a/xen/arch/x86/pqos.c > > +++ b/xen/arch/x86/pqos.c > > @@ -20,6 +20,7 @@ > > */ > > #include <asm/processor.h> > > #include <xen/init.h> > > +#include <xen/spinlock.h> > > #include <asm/pqos.h> > > > > static bool_t pqos_enabled = 1; > > @@ -29,6 +30,7 @@ unsigned int cqm_res_count = 0; > > unsigned int cqm_upscaling_factor = 0; > > bool_t cqm_enabled = 0; > > struct cqm_res_struct *cqm_res_array = NULL; > > +static DEFINE_SPINLOCK(cqm_lock); > > > > static void init_cqm(void) > > { > > @@ -78,6 +80,62 @@ void init_platform_qos(void) > > init_qos_monitor(); > > } > > > > +bool_t system_support_cqm(void) > > "system_supports_cqm"OK> > > +{ > > + return cqm_enabled; > > +} > > + > > +int alloc_cqm_resource(domid_t domain_id) > > Perhaps alloc_rmid() given its current semantics? > > > +{ > > + int i, rmid = -1; > > + unsigned long flags; > > + > > + if ( !system_support_cqm() ) > > + goto out; > > Probably better as an ASSERT(). The caller should really not try to > allocate cqm resources if the system doesn''t have cqm enabled.OK.> > > + > > + spin_lock_irqsave(&cqm_lock, flags); > > + for ( i = 0; i < cqm_res_count; i++ ) > > + { > > + if ( cqm_res_array[i].inuse) > > + continue; > > + > > + rmid = i; > > + cqm_res_array[rmid].inuse = 1; > > + cqm_res_array[rmid].domain_id = domain_id; > > + break; > > + } > > + spin_unlock_irqrestore(&cqm_lock, flags); > > + > > + /* No CQM RMID available, assign RMID=0 by default */ > > + if ( rmid == -1 ) > > + rmid = 0; > > + > > +out: > > + return rmid; > > +} > > + > > +void free_cqm_resource(domid_t domain_id) > > +{ > > + int rmid; > > + unsigned long flags; > > + > > + if ( !system_support_cqm() ) > > + return; > > + > > + spin_lock_irqsave(&cqm_lock, flags); > > + /* CQM RMID 0 is reserved, enumerate from 1 */ > > + for ( rmid = 1; rmid < cqm_res_count; rmid++ ) > > + { > > + if ( cqm_res_array[rmid].domain_id != domain_id ) > > + continue; > > + > > + cqm_res_array[rmid].inuse = 0; > > + cqm_res_array[rmid].domain_id = 0; > > + break; > > + } > > + spin_unlock_irqrestore(&cqm_lock, flags); > > Can you not use d->arch.pqos_cqm_rmid here to save the search ?This needs to pass the "d" as a parameter, but seems a good idea. Will reflect above suggestions in next version patch. Thanks, Dongxiao> > > +} > > + > > /* > > * Local variables: > > * mode: C > > diff --git a/xen/common/domctl.c b/xen/common/domctl.c > > index 904d27b..1c2e320 100644 > > --- a/xen/common/domctl.c > > +++ b/xen/common/domctl.c > > @@ -425,7 +425,8 @@ long > do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) > > | XEN_DOMCTL_CDF_pvh_guest > > | XEN_DOMCTL_CDF_hap > > | XEN_DOMCTL_CDF_s3_integrity > > - | XEN_DOMCTL_CDF_oos_off)) ) > > + | XEN_DOMCTL_CDF_oos_off > > + | XEN_DOMCTL_CDF_pqos_cqm)) ) > > break; > > > > dom = op->domain; > > @@ -467,6 +468,8 @@ long > do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) > > domcr_flags |= DOMCRF_s3_integrity; > > if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off ) > > domcr_flags |= DOMCRF_oos_off; > > + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_pqos_cqm ) > > + domcr_flags |= DOMCRF_pqos_cqm; > > > > d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref); > > if ( IS_ERR(d) ) > > diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h > > index 9d39061..b0479aa 100644 > > --- a/xen/include/asm-x86/domain.h > > +++ b/xen/include/asm-x86/domain.h > > @@ -313,6 +313,8 @@ struct arch_domain > > spinlock_t e820_lock; > > struct e820entry *e820; > > unsigned int nr_e820; > > + > > + int pqos_cqm_rmid; /* CQM RMID assigned to the domain */ > > } __cacheline_aligned; > > > > #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list)) > > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > > index 2184ea9..7e32fa5 100644 > > --- a/xen/include/asm-x86/pqos.h > > +++ b/xen/include/asm-x86/pqos.h > > @@ -34,4 +34,8 @@ struct cqm_res_struct { > > > > void init_platform_qos(void); > > > > +bool_t system_support_cqm(void); > > +int alloc_cqm_resource(domid_t); > > +void free_cqm_resource(domid_t); > > + > > #endif > > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > > index 01a3652..47a850a 100644 > > --- a/xen/include/public/domctl.h > > +++ b/xen/include/public/domctl.h > > @@ -62,6 +62,9 @@ struct xen_domctl_createdomain { > > /* Is this a PVH guest (as opposed to an HVM or PV guest)? */ > > #define _XEN_DOMCTL_CDF_pvh_guest 4 > > #define XEN_DOMCTL_CDF_pvh_guest > (1U<<_XEN_DOMCTL_CDF_pvh_guest) > > + /* Enable pqos-cqm? */ > > +#define _XEN_DOMCTL_CDF_pqos_cqm 5 > > +#define XEN_DOMCTL_CDF_pqos_cqm > (1U<<_XEN_DOMCTL_CDF_pqos_cqm) > > uint32_t flags; > > }; > > typedef struct xen_domctl_createdomain xen_domctl_createdomain_t; > > diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h > > index cbdf377..3a42656 100644 > > --- a/xen/include/xen/sched.h > > +++ b/xen/include/xen/sched.h > > @@ -507,6 +507,9 @@ struct domain *domain_create( > > /* DOMCRF_pvh: Create PV domain in HVM container. */ > > #define _DOMCRF_pvh 5 > > #define DOMCRF_pvh (1U<<_DOMCRF_pvh) > > + /* DOMCRF_pqos_cqm: Create a domain with CQM support */ > > +#define _DOMCRF_pqos_cqm 6 > > +#define DOMCRF_pqos_cqm (1U<<_DOMCRF_pqos_cqm) > > > > /* > > * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
Xu, Dongxiao
2013-Nov-20 13:19 UTC
Re: [PATCH 4/8] x86: dynamically attach/detach CQM service for a guest
> -----Original Message----- > From: Andrew Cooper [mailto:andrew.cooper3@citrix.com] > Sent: Wednesday, November 20, 2013 7:45 PM > To: Xu, Dongxiao > Cc: xen-devel@lists.xen.org > Subject: Re: [Xen-devel] [PATCH 4/8] x86: dynamically attach/detach CQM service > for a guest > > On 20/11/13 03:27, dongxiao.xu@intel.com wrote: > > From: Dongxiao Xu <dongxiao.xu@intel.com> > > > > Add hypervisor side support for dynamically attach and detach CQM > > services for a certain guest. > > > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > > --- > > xen/arch/x86/domctl.c | 66 > +++++++++++++++++++++++++++++++++++++++++++ > > xen/include/public/domctl.h | 15 ++++++++++ > > 2 files changed, 81 insertions(+) > > > > diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c > > index f7e4586..df6a373 100644 > > --- a/xen/arch/x86/domctl.c > > +++ b/xen/arch/x86/domctl.c > > @@ -35,6 +35,7 @@ > > #include <asm/mem_sharing.h> > > #include <asm/xstate.h> > > #include <asm/debugger.h> > > +#include <asm/pqos.h> > > > > static int gdbsx_guest_mem_io( > > domid_t domid, struct xen_domctl_gdbsx_memio *iop) > > @@ -1223,6 +1224,71 @@ long arch_do_domctl( > > } > > break; > > > > + case XEN_DOMCTL_attach_pqos: > > + { > > + struct domain *d; > > + > > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) > > + { > > + ret = -EBUSY; > > + break; > > + } > > do_domctl has done this for us. There is also no need to shadow ''d''.OK, will remove the logic.> > > + > > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) > > + { > > + if ( !system_support_cqm() ) > > + ret = -ENODEV; > > Surely this is generic, rather than condition on the caller specifying > XEN_DOMCTL_ADF_pqos_cqmCQM is just one type of monitoring resource, there may be other types later.> > > + else if ( d->arch.pqos_cqm_rmid > 0 ) > > + ret = -EINVAL; > > + else > > + { > > + ret = 0; > > + d->arch.pqos_cqm_rmid > alloc_cqm_resource(d->domain_id); > > + if ( d->arch.pqos_cqm_rmid <= 0 ) > > + { > > + /* set to default value */ > > + d->arch.pqos_cqm_rmid = 0; > > + ret = -ENODEV; > > Perhaps E2BIG or EBUSY ? This error can be fixed by detaching cqm from > other domains.OK, maybe EBUSY is a better choice.> > > + } > > + } > > + } > > + else > > + ret = -EINVAL; > > + > > + rcu_unlock_domain(d); > > + } > > + break; > > + > > + case XEN_DOMCTL_detach_pqos: > > + { > > + struct domain *d; > > + > > + if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL ) > > + { > > + ret = -EBUSY; > > + break; > > + } > > + > > + if ( domctl->u.qos_res.flags & XEN_DOMCTL_ADF_pqos_cqm ) > > + { > > + if ( !system_support_cqm() ) > > + ret = -ENODEV; > > + else if ( d->arch.pqos_cqm_rmid > 0 ) > > + { > > + free_cqm_resource(d->domain_id); > > + d->arch.pqos_cqm_rmid = 0; > > + ret = 0; > > + } > > + else > > + ret = -EINVAL; > > + } > > + else > > + ret = -EINVAL; > > + > > + rcu_unlock_domain(d); > > + } > > + break; > > + > > default: > > ret = iommu_do_domctl(domctl, d, u_domctl); > > break; > > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > > index 47a850a..4fe21db 100644 > > --- a/xen/include/public/domctl.h > > +++ b/xen/include/public/domctl.h > > @@ -872,6 +872,17 @@ struct xen_domctl_set_max_evtchn { > > typedef struct xen_domctl_set_max_evtchn > xen_domctl_set_max_evtchn_t; > > DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_max_evtchn_t); > > > > +/* XEN_DOMCTL_attach_pqos */ > > +/* XEN_DOMCTL_detach_pqos */ > > +struct xen_domctl_qos_resource { > > + /* Attach or detach flag for cqm */ > > +#define _XEN_DOMCTL_ADF_pqos_cqm 0 > > +#define XEN_DOMCTL_ADF_pqos_cqm > (1U<<_XEN_DOMCTL_ADF_pqos_cqm) > > + uint32_t flags; > > The implementation semantics of this flag is redundant. You > differentiate attach/detach both on the domctl subop, and the flag in > this structure, leaving two invalid states. > > It might be better to have a XEN_DOMCTL_qpos_op and a sub command, if > more of these hypercalls are on their way.Flag is to identify the type of QoS monitoring resource, and CQM is one example.> > > +}; > > +typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; > > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); > > + > > struct xen_domctl { > > uint32_t cmd; > > #define XEN_DOMCTL_createdomain 1 > > @@ -941,6 +952,9 @@ struct xen_domctl { > > #define XEN_DOMCTL_setnodeaffinity 68 > > #define XEN_DOMCTL_getnodeaffinity 69 > > #define XEN_DOMCTL_set_max_evtchn 70 > > +#define XEN_DOMCTL_attach_pqos 71 > > +#define XEN_DOMCTL_detach_pqos 72 > > +#define XEN_DOMCTL_list_pqos 73 > > You introduce this list_pqos domctl with no implementation.Yes, it should be in the patch 6/8. Thanks, Dongxiao> > > #define XEN_DOMCTL_gdbsx_guestmemio 1000 > > #define XEN_DOMCTL_gdbsx_pausevcpu 1001 > > #define XEN_DOMCTL_gdbsx_unpausevcpu 1002 > > @@ -1001,6 +1015,7 @@ struct xen_domctl { > > struct xen_domctl_set_broken_page_p2m set_broken_page_p2m; > > struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu; > > struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; > > + struct xen_domctl_qos_resource qos_res; > > uint8_t pad[128]; > > } u; > > };
> -----Original Message----- > From: Andrew Cooper [mailto:andrew.cooper3@citrix.com] > Sent: Wednesday, November 20, 2013 7:57 PM > To: Xu, Dongxiao > Cc: xen-devel@lists.xen.org > Subject: Re: [Xen-devel] [PATCH 6/8] x86: get per domain CQM information > > On 20/11/13 03:27, dongxiao.xu@intel.com wrote: > > From: Dongxiao Xu <dongxiao.xu@intel.com> > > > > Retrive CQM information for certain domain, which reflects the L3 cache > > occupancy for a socket. > > > > Signed-off-by: Jiongxi Li <jiongxi.li@intel.com> > > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com> > > --- > > xen/arch/x86/pqos.c | 60 > ++++++++++++++++++++++++++++++++++++ > > xen/arch/x86/sysctl.c | 64 > +++++++++++++++++++++++++++++++++++++++ > > xen/include/asm-x86/msr-index.h | 4 +++ > > xen/include/asm-x86/pqos.h | 14 +++++++++ > > xen/include/public/domctl.h | 14 +++++++++ > > xen/include/public/sysctl.h | 23 ++++++++++++++ > > 6 files changed, 179 insertions(+) > > > > diff --git a/xen/arch/x86/pqos.c b/xen/arch/x86/pqos.c > > index 895d892..3699efe 100644 > > --- a/xen/arch/x86/pqos.c > > +++ b/xen/arch/x86/pqos.c > > @@ -19,13 +19,30 @@ > > * Place - Suite 330, Boston, MA 02111-1307 USA. > > */ > > #include <asm/processor.h> > > +#include <asm/msr.h> > > +#include <xen/cpumask.h> > > #include <xen/init.h> > > #include <xen/spinlock.h> > > +#include <public/domctl.h> > > #include <asm/pqos.h> > > > > static bool_t pqos_enabled = 1; > > boolean_param("pqos", pqos_enabled); > > > > +static void read_qm_data(void *arg) > > +{ > > + struct qm_element *qm_element = arg; > > + > > + wrmsr(MSR_IA32_QOSEVTSEL, qm_element->evtid, > qm_element->rmid); > > + rdmsrl(MSR_IA32_QMC, qm_element->qm_data); > > +} > > + > > +static void get_generic_qm_info(struct qm_element *qm_element) > > +{ > > + int cpu = qm_element->cpu; > > + on_selected_cpus(cpumask_of(cpu), read_qm_data, qm_element, 1); > > +} > > + > > unsigned int cqm_res_count = 0; > > unsigned int cqm_upscaling_factor = 0; > > bool_t cqm_enabled = 0; > > @@ -85,6 +102,25 @@ bool_t system_support_cqm(void) > > return cqm_enabled; > > } > > > > +unsigned int get_cqm_count(void) > > +{ > > + return cqm_res_count; > > +} > > + > > +unsigned int get_cqm_avail(void) > > +{ > > + unsigned int cqm_avail = 0; > > + int i; > > + > > + for (i = 0; i < cqm_res_count; i++) > > + { > > + if ( !cqm_res_array[i].inuse ) > > + cqm_avail++; > > + } > > Style - extra spaces inside brackets for the for loop, and these braces > can go.OK.> > > + > > + return cqm_avail; > > +} > > + > > int alloc_cqm_resource(domid_t domain_id) > > { > > int i, rmid = -1; > > @@ -136,6 +172,30 @@ void free_cqm_resource(domid_t domain_id) > > spin_unlock_irqrestore(&cqm_lock, flags); > > } > > > > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, > > + struct xen_domctl_getdomcqminfo *info) > > +{ > > + struct qm_element element; > > + int cpu, i; > > + > > + for_each_cpu ( cpu, &cpu_cqmdata_map ) > > + { > > + element.cpu = cpu; > > + element.rmid = rmid; > > + element.evtid = QOS_MONITOR_EVTID_L3; > > + > > + get_generic_qm_info(&element); > > + > > + i = cpu_to_socket(cpu); > > + info->socket_cqmdata[i].valid > > + (element.qm_data & IA32_QM_CTR_ERROR_MASK) ? 0 : 1; > > + if ( info->socket_cqmdata[i].valid ) > > + info->socket_cqmdata[i].l3c_occupancy = element.qm_data * > cqm_upscaling_factor; > > + else > > + info->socket_cqmdata[i].l3c_occupancy = 0; > > + } > > +} > > + > > /* > > * Local variables: > > * mode: C > > diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c > > index 15d4b91..a779fdc 100644 > > --- a/xen/arch/x86/sysctl.c > > +++ b/xen/arch/x86/sysctl.c > > @@ -28,6 +28,7 @@ > > #include <xen/nodemask.h> > > #include <xen/cpu.h> > > #include <xsm/xsm.h> > > +#include <asm/pqos.h> > > > > #define get_xen_guest_handle(val, hnd) do { val = (hnd).p; } while (0) > > > > @@ -101,6 +102,69 @@ long arch_do_sysctl( > > } > > break; > > > > + case XEN_SYSCTL_getdomcqminfolist: > > + { > > + struct domain *d; > > + struct xen_domctl_getdomcqminfo info; > > + uint32_t resource_count; > > + uint32_t resource_avail; > > + uint32_t num_domains = 0; > > + cpumask_t cpu_cqmdata_map; > > + DECLARE_BITMAP(sockets, QOS_MAX_SOCKETS); > > + int cpu; > > unsigned int.OK.> > > + > > + if ( !system_support_cqm() ) > > + { > > + ret = -EFAULT; > > ENODEV surely ?OK.> > > + break; > > + } > > + > > + resource_count = get_cqm_count(); > > + resource_avail = get_cqm_avail(); > > + > > + cpumask_clear(&cpu_cqmdata_map); > > + bitmap_zero(sockets, QOS_MAX_SOCKETS); > > + for_each_online_cpu(cpu) > > + { > > + int i = cpu_to_socket(cpu); > > + if ( test_and_set_bit(i, sockets) ) > > + continue; > > + cpumask_set_cpu(cpu, &cpu_cqmdata_map); > > + } > > + > > + rcu_read_lock(&domlist_read_lock); > > + for_each_domain ( d ) > > + { > > + if ( d->domain_id < sysctl->u.getdomaininfolist.first_domain ) > > + continue; > > + if ( num_domains == sysctl->u.getdomaininfolist.max_domains ) > > + break; > > + if ( d->arch.pqos_cqm_rmid <= 0 ) > > + continue; > > + memset(&info, 0, sizeof(struct xen_domctl_getdomcqminfo)); > > + info.domain = d->domain_id; > > + get_cqm_info(d->arch.pqos_cqm_rmid, cpu_cqmdata_map, > &info); > > + > > + if ( copy_to_guest_offset(sysctl->u.getdomcqminfolist.buffer, > > + num_domains, &info, 1) ) > > + { > > + ret = -EFAULT; > > + break; > > + } > > + > > + num_domains++; > > + } > > + rcu_read_unlock(&domlist_read_lock); > > + > > + sysctl->u.getdomcqminfolist.num_domains = num_domains; > > + sysctl->u.getdomcqminfolist.resource_count = resource_count; > > + sysctl->u.getdomcqminfolist.resource_avail = resource_avail; > > + > > + if ( copy_to_guest(u_sysctl, sysctl, 1) ) > > + ret = -EFAULT; > > + } > > + break; > > + > > default: > > ret = -ENOSYS; > > break; > > diff --git a/xen/include/asm-x86/msr-index.h > b/xen/include/asm-x86/msr-index.h > > index e597a28..46ef165 100644 > > --- a/xen/include/asm-x86/msr-index.h > > +++ b/xen/include/asm-x86/msr-index.h > > @@ -488,4 +488,8 @@ > > /* Geode defined MSRs */ > > #define MSR_GEODE_BUSCONT_CONF0 0x00001900 > > > > +/* Platform QoS register */ > > +#define MSR_IA32_QOSEVTSEL 0x00000c8d > > +#define MSR_IA32_QMC 0x00000c8e > > + > > #endif /* __ASM_MSR_INDEX_H */ > > diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h > > index 7e32fa5..6d1b1e8 100644 > > --- a/xen/include/asm-x86/pqos.h > > +++ b/xen/include/asm-x86/pqos.h > > @@ -27,15 +27,29 @@ > > /* QoS Monitoring Event ID */ > > #define QOS_MONITOR_EVTID_L3 0x1 > > > > +/* IA32_QM_CTR */ > > +#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62) > > + > > struct cqm_res_struct { > > bool_t inuse; > > uint16_t domain_id; > > }; > > > > +struct qm_element { > > + uint16_t cpu; > > + uint32_t rmid; > > + uint8_t evtid; > > + uint64_t qm_data; > > +}; > > + > > The packing of this structure is rather poor. Please re-order to reduce > some of the holes. > > Also, cpu parameters are unsigned int in Xen.Both OK.> > > void init_platform_qos(void); > > > > bool_t system_support_cqm(void); > > int alloc_cqm_resource(domid_t); > > void free_cqm_resource(domid_t); > > +unsigned int get_cqm_count(void); > > +unsigned int get_cqm_avail(void); > > +void get_cqm_info(uint32_t rmid, cpumask_t cpu_cqmdata_map, > > + struct xen_domctl_getdomcqminfo *info); > > > > #endif > > diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h > > index 4fe21db..bdefc83 100644 > > --- a/xen/include/public/domctl.h > > +++ b/xen/include/public/domctl.h > > @@ -883,6 +883,20 @@ struct xen_domctl_qos_resource { > > typedef struct xen_domctl_qos_resource xen_domctl_qos_resource_t; > > DEFINE_XEN_GUEST_HANDLE(xen_domctl_qos_resource_t); > > > > +struct xen_socket_cqmdata { > > + uint8_t valid; > > + uint64_t l3c_occupancy; > > +}; > > + > > +struct xen_domctl_getdomcqminfo { > > + /* OUT variables. */ > > + domid_t domain; > > +#define QOS_MAX_SOCKETS 128 > > + struct xen_socket_cqmdata socket_cqmdata[QOS_MAX_SOCKETS]; > > +}; > > +typedef struct xen_domctl_getdomcqminfo xen_domctl_getdomcqminfo_t; > > +DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomcqminfo_t); > > + > > struct xen_domctl { > > uint32_t cmd; > > #define XEN_DOMCTL_createdomain 1 > > diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h > > index 8437d31..91f206e 100644 > > --- a/xen/include/public/sysctl.h > > +++ b/xen/include/public/sysctl.h > > @@ -149,6 +149,14 @@ struct xen_sysctl_perfc_op { > > typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t; > > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t); > > > > +struct xen_sysctl_getcqminfo > > +{ > > + uint32_t resource_count; > > + uint32_t resource_avail; > > + struct xen_domctl_getdomcqminfo *dom_cqminfo; > > XEN_GUEST_HANDLE please.OK. Will reflect above suggestions in next version changes. Thanks, Dongxiao> > > +}; > > +typedef struct xen_sysctl_getcqminfo xen_sysctl_getcqminfo_t; > > + > > /* XEN_SYSCTL_getdomaininfolist */ > > struct xen_sysctl_getdomaininfolist { > > /* IN variables. */ > > @@ -632,6 +640,19 @@ struct xen_sysctl_coverage_op { > > typedef struct xen_sysctl_coverage_op xen_sysctl_coverage_op_t; > > DEFINE_XEN_GUEST_HANDLE(xen_sysctl_coverage_op_t); > > > > +/* XEN_SYSCTL_getdomcqminfolist */ > > +struct xen_sysctl_getdomcqminfolist { > > + /* IN variables. */ > > + domid_t first_domain; > > + uint32_t max_domains; > > + XEN_GUEST_HANDLE_64(xen_domctl_getdomcqminfo_t) buffer; > > + /* OUT variables. */ > > + uint32_t num_domains; > > + uint32_t resource_count; > > + uint32_t resource_avail; > > +}; > > +typedef struct xen_sysctl_getdomcqminfolist > xen_sysctl_getdomcqminfolist_t; > > +DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomcqminfolist_t); > > > > struct xen_sysctl { > > uint32_t cmd; > > @@ -654,6 +675,7 @@ struct xen_sysctl { > > #define XEN_SYSCTL_cpupool_op 18 > > #define XEN_SYSCTL_scheduler_op 19 > > #define XEN_SYSCTL_coverage_op 20 > > +#define XEN_SYSCTL_getdomcqminfolist 21 > > uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */ > > union { > > struct xen_sysctl_readconsole readconsole; > > @@ -675,6 +697,7 @@ struct xen_sysctl { > > struct xen_sysctl_cpupool_op cpupool_op; > > struct xen_sysctl_scheduler_op scheduler_op; > > struct xen_sysctl_coverage_op coverage_op; > > + struct xen_sysctl_getdomcqminfolist getdomcqminfolist; > > uint8_t pad[128]; > > } u; > > };