This patch extends the CONFIG_HOTPLUG_CPU behavior down into the
hypervisor. Currently when a CPU in Linux is moved offline,
echo 0 > /sys/devices/system/cpu/cpuX/online
the offline cpu yields its slice back to the hypervisor. This patch
adds two SCHEDOPS (vcpu_down/vcpu_up) which set/clear a new VCPU flag,
VCPU_down. The domain_runnable() check now looks at this flag and
subsequently the vcpu is not scheduled when VCPU_down is set.
The patch was built and tested against 20050606 nightly snapshot.
Testing requires DOMU with CONFIG_SMP and CONFIG_HOTPLUG_CPU. Please
apply.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c | 7 +
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 4
linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h | 31 +++++++
xen/common/schedule.c | 48 +++++++++++
xen/include/public/xen.h | 3
xen/include/xen/sched.h | 5 -
6 files changed, 96 insertions(+), 2 deletions(-)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c
c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-05
22:09:07.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-06
15:15:34.184156730 -0500
@@ -154,8 +154,13 @@
cpu_clear(cpu, cpu_idle_map);
rmb();
- if (cpu_is_offline(cpu))
+ if (cpu_is_offline(cpu)) {
+#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU)
+ /* Tell hypervisor not to schedule dead vcpus */
+ HYPERVISOR_vcpu_down(cpu);
+#endif
play_dead();
+ }
irq_stat[cpu].idle_timestamp = jiffies;
xen_idle();
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-05
22:09:13.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-06
15:44:15.988564794 -0500
@@ -1380,6 +1380,10 @@
}
#ifdef CONFIG_HOTPLUG_CPU
+#ifdef CONFIG_XEN
+ /* Tell hypervisor to bring vcpu up */
+ HYPERVISOR_vcpu_up(cpu);
+#endif
/* Already up, and in cpu_quiescent now? */
if (cpu_isset(cpu, smp_commenced_mask)) {
cpu_enable(cpu);
diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h
c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h
--- b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-05
22:09:15.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-06
14:18:17.414798645 -0500
@@ -517,4 +517,35 @@
return ret;
}
+static inline int
+HYPERVISOR_vcpu_down(
+ int vcpu)
+{
+ int ret;
+ unsigned long ign1;
+ __asm__ __volatile__ (
+ TRAP_INSTR
+ : "=a" (ret), "=b" (ign1)
+ : "0" (__HYPERVISOR_sched_op),
+ "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift))
+ : "memory" );
+
+ return ret;
+}
+
+static inline int
+HYPERVISOR_vcpu_up(
+ int vcpu)
+{
+ int ret;
+ unsigned long ign1;
+ __asm__ __volatile__ (
+ TRAP_INSTR
+ : "=a" (ret), "=b" (ign1)
+ : "0" (__HYPERVISOR_sched_op),
+ "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift))
+ : "memory" );
+
+ return ret;
+}
#endif /* __HYPERCALL_H__ */
diff -urN b/xen/common/schedule.c c/xen/common/schedule.c
--- b/xen/common/schedule.c 2005-06-05 22:09:14.000000000 -0500
+++ c/xen/common/schedule.c 2005-06-06 15:50:35.549218230 -0500
@@ -261,6 +261,44 @@
return 0;
}
+/* Mark target vcpu as non-runnable so it is not scheduled */
+static long do_vcpu_down(int vcpu)
+{
+ struct vcpu *target;
+
+ if (vcpu > MAX_VIRT_CPUS)
+ return -EINVAL;
+
+ target = current->domain->vcpu[vcpu];
+ /* DEBUG
+ * printk("DOM%d VCPU%d going down\n",
+ * target->domain->domain_id, target->vcpu_id);
+ */
+ set_bit(_VCPUF_down, &target->vcpu_flags);
+
+ return 0;
+}
+
+/* Mark target vcpu as runnable and wake it */
+static long do_vcpu_up(int vcpu)
+{
+ struct vcpu *target;
+
+ if (vcpu > MAX_VIRT_CPUS)
+ return -EINVAL;
+
+ target = current->domain->vcpu[vcpu];
+ /* DEBUG
+ * printk("DOM%d VCPU%d coming up\n",
+ * target->domain->domain_id, target->vcpu_id);
+ */
+ clear_bit(_VCPUF_down, &target->vcpu_flags);
+ /* wake vcpu */
+ domain_wake(target);
+
+ return 0;
+}
+
/*
* Demultiplex scheduler-related hypercalls.
*/
@@ -290,6 +328,16 @@
domain_shutdown((u8)(op >> SCHEDOP_reasonshift));
break;
}
+ case SCHEDOP_vcpu_down:
+ {
+ ret = do_vcpu_down((int)(op >> SCHEDOP_vcpushift));
+ break;
+ }
+ case SCHEDOP_vcpu_up:
+ {
+ ret = do_vcpu_up((int)(op >> SCHEDOP_vcpushift));
+ break;
+ }
default:
ret = -ENOSYS;
diff -urN b/xen/include/public/xen.h c/xen/include/public/xen.h
--- b/xen/include/public/xen.h 2005-06-05 22:09:13.000000000 -0500
+++ c/xen/include/public/xen.h 2005-06-06 14:18:17.427796825 -0500
@@ -200,8 +200,11 @@
#define SCHEDOP_yield 0 /* Give up the CPU voluntarily. */
#define SCHEDOP_block 1 /* Block until an event is received. */
#define SCHEDOP_shutdown 2 /* Stop executing this domain. */
+#define SCHEDOP_vcpu_down 3 /* make target VCPU not-runnable. */
+#define SCHEDOP_vcpu_up 4 /* make target VCPU runnable. */
#define SCHEDOP_cmdmask 255 /* 8-bit command. */
#define SCHEDOP_reasonshift 8 /* 8-bit reason code. (SCHEDOP_shutdown) */
+#define SCHEDOP_vcpushift 8 /* 8-bit VCPU target. (SCHEDOP_up|down) */
/*
* Reason codes for SCHEDOP_shutdown. These may be interpreted by control
diff -urN b/xen/include/xen/sched.h c/xen/include/xen/sched.h
--- b/xen/include/xen/sched.h 2005-06-05 22:09:08.000000000 -0500
+++ c/xen/include/xen/sched.h 2005-06-06 14:18:17.000000000 -0500
@@ -346,6 +346,9 @@
/* Initialization completed. */
#define _VCPUF_initialised 8
#define VCPUF_initialised (1UL<<_VCPUF_initialised)
+ /* VCPU is not-runnable */
+#define _VCPUF_down 9
+#define VCPUF_down (1UL<<_VCPUF_down)
/*
* Per-domain flags (domain_flags).
@@ -375,7 +378,7 @@
static inline int domain_runnable(struct vcpu *v)
{
return ( (atomic_read(&v->pausecnt) == 0) &&
- !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause))
&&
+ !(v->vcpu_flags &
(VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) &&
!(v->domain->domain_flags &
(DOMF_shutdown|DOMF_shuttingdown)) );
}
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel
Ryan Harper
2005-Jun-08 21:29 UTC
[Xen-devel] [PATCH][RESUBMIT] don''t schedule unplugged vcpus
* Ryan Harper <ryanh@us.ibm.com> [2005-06-06 16:04]:> This patch extends the CONFIG_HOTPLUG_CPU behavior down into the > hypervisor. Currently when a CPU in Linux is moved offline, > > echo 0 > /sys/devices/system/cpu/cpuX/online > > the offline cpu yields its slice back to the hypervisor. This patch > adds two SCHEDOPS (vcpu_down/vcpu_up) which set/clear a new VCPU flag, > VCPU_down. The domain_runnable() check now looks at this flag and > subsequently the vcpu is not scheduled when VCPU_down is set. > > The patch was built and tested against 20050606 nightly snapshot. > Testing requires DOMU with CONFIG_SMP and CONFIG_HOTPLUG_CPU. Please > apply.I''ve added in changes to DOM0_GETDOMINFO and DOM0_GETVCPUCONTEXT hypercalls. dominfo now creates a vcpu_online_map bitmap which marks whether vcpus are up or down. I didn''t want to clobber either the total number of vcpus allocated to a domain (n_vcpu) nor the vcpu_to_cpu mapping since both are still valid whether the vcpu is being scheduled or not. I modified vcpucontext to give the context for the first vcpu not down. If the requested vcpu is down, it will return the context of the next vcpu that is up, or -ESRCH if no vcpu past the requested vcpu is valid. I modified xm list -v to display an ONLINE column which indicates the online status of each vcpu in a domain. Please apply. -- Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 ryanh@us.ibm.com diffstat output: linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c | 7 + linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 4 linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h | 31 +++++++ tools/libxc/xc.h | 1 tools/libxc/xc_domain.c | 1 tools/python/xen/lowlevel/xc/xc.c | 7 + tools/python/xen/xend/XendDomainInfo.py | 2 tools/python/xen/xm/main.py | 6 - xen/common/dom0_ops.c | 17 +++ xen/common/schedule.c | 48 +++++++++++ xen/include/public/dom0_ops.h | 3 xen/include/public/xen.h | 3 xen/include/xen/sched.h | 5 - 13 files changed, 127 insertions(+), 8 deletions(-) Signed-off-by: Ryan Harper <ryanh@us.ibm.com> --- diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c --- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-07 22:09:53.000000000 -0500 +++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-08 12:57:58.658741356 -0500 @@ -154,8 +154,13 @@ cpu_clear(cpu, cpu_idle_map); rmb(); - if (cpu_is_offline(cpu)) + if (cpu_is_offline(cpu)) { +#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU) + /* Tell hypervisor not to schedule dead vcpus */ + HYPERVISOR_vcpu_down(cpu); +#endif play_dead(); + } irq_stat[cpu].idle_timestamp = jiffies; xen_idle(); diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c --- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-07 22:10:00.000000000 -0500 +++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-08 12:57:58.663740663 -0500 @@ -1380,6 +1380,10 @@ } #ifdef CONFIG_HOTPLUG_CPU +#ifdef CONFIG_XEN + /* Tell hypervisor to bring vcpu up */ + HYPERVISOR_vcpu_up(cpu); +#endif /* Already up, and in cpu_quiescent now? */ if (cpu_isset(cpu, smp_commenced_mask)) { cpu_enable(cpu); diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h --- b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-07 22:10:03.000000000 -0500 +++ c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-08 12:57:58.664740524 -0500 @@ -517,4 +517,35 @@ return ret; } +static inline int +HYPERVISOR_vcpu_down( + int vcpu) +{ + int ret; + unsigned long ign1; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret), "=b" (ign1) + : "0" (__HYPERVISOR_sched_op), + "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift)) + : "memory" ); + + return ret; +} + +static inline int +HYPERVISOR_vcpu_up( + int vcpu) +{ + int ret; + unsigned long ign1; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret), "=b" (ign1) + : "0" (__HYPERVISOR_sched_op), + "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift)) + : "memory" ); + + return ret; +} #endif /* __HYPERCALL_H__ */ diff -urN b/tools/libxc/xc.h c/tools/libxc/xc.h --- b/tools/libxc/xc.h 2005-06-07 22:09:56.000000000 -0500 +++ c/tools/libxc/xc.h 2005-06-08 12:57:58.000000000 -0500 @@ -118,6 +118,7 @@ u64 cpu_time; unsigned long max_memkb; unsigned int vcpus; + u32 vcpu_online_map; s32 vcpu_to_cpu[MAX_VIRT_CPUS]; cpumap_t cpumap[MAX_VIRT_CPUS]; } xc_dominfo_t; diff -urN b/tools/libxc/xc_domain.c c/tools/libxc/xc_domain.c --- b/tools/libxc/xc_domain.c 2005-06-07 22:09:59.000000000 -0500 +++ c/tools/libxc/xc_domain.c 2005-06-08 12:57:58.000000000 -0500 @@ -106,6 +106,7 @@ info->shared_info_frame = op.u.getdomaininfo.shared_info_frame; info->cpu_time = op.u.getdomaininfo.cpu_time; info->vcpus = op.u.getdomaininfo.n_vcpu; + info->vcpu_online_map = op.u.getdomaininfo.vcpu_online_map; memcpy(&info->vcpu_to_cpu, &op.u.getdomaininfo.vcpu_to_cpu, sizeof(info->vcpu_to_cpu)); memcpy(&info->cpumap, &op.u.getdomaininfo.cpumap, diff -urN b/tools/python/xen/lowlevel/xc/xc.c c/tools/python/xen/lowlevel/xc/xc.c --- b/tools/python/xen/lowlevel/xc/xc.c 2005-06-07 22:09:59.000000000 -0500 +++ c/tools/python/xen/lowlevel/xc/xc.c 2005-06-08 14:40:13.739556324 -0500 @@ -199,7 +199,7 @@ PyObject *kwds) { XcObject *xc = (XcObject *)self; - PyObject *list, *vcpu_list, *cpumap_list, *info_dict; + PyObject *list, *vcpu_list, *online_list, *cpumap_list, *info_dict; u32 first_dom = 0; int max_doms = 1024, nr_doms, i, j; @@ -221,11 +221,15 @@ { vcpu_list = PyList_New(MAX_VIRT_CPUS); cpumap_list = PyList_New(MAX_VIRT_CPUS); + online_list = PyList_New(MAX_VIRT_CPUS); for ( j = 0; j < MAX_VIRT_CPUS; j++ ) { PyList_SetItem( vcpu_list, j, Py_BuildValue("i", info[i].vcpu_to_cpu[j])); PyList_SetItem( cpumap_list, j, Py_BuildValue("i", info[i].cpumap[j])); + PyList_SetItem( online_list, j, + Py_BuildValue("i", + ((1<<j) & info[i].vcpu_online_map)>>j)); } info_dict = Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i" @@ -244,6 +248,7 @@ "shutdown_reason", info[i].shutdown_reason); PyDict_SetItemString( info_dict, "vcpu_to_cpu", vcpu_list ); PyDict_SetItemString( info_dict, "cpumap", cpumap_list ); + PyDict_SetItemString( info_dict, "vcpu_online_map", online_list ); PyList_SetItem( list, i, info_dict); } diff -urN b/tools/python/xen/xend/XendDomainInfo.py c/tools/python/xen/xend/XendDomainInfo.py --- b/tools/python/xen/xend/XendDomainInfo.py 2005-06-07 22:09:59.000000000 -0500 +++ c/tools/python/xen/xend/XendDomainInfo.py 2005-06-08 12:57:58.000000000 -0500 @@ -393,6 +393,8 @@ sxpr.append([''cpu_time'', self.info[''cpu_time'']/1e9]) sxpr.append([''vcpus'', self.info[''vcpus'']]) sxpr.append([''cpumap'', self.info[''cpumap'']]) + sxpr.append([''vcpu_online_map'', ''''.join(map(lambda x: str(x), + self.info[''vcpu_online_map''][0:self.info[''vcpus'']]))]) sxpr.append([''vcpu_to_cpu'', ''''.join(map(lambda x: str(x), self.info[''vcpu_to_cpu''][0:self.info[''vcpus'']]))]) diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py --- b/tools/python/xen/xm/main.py 2005-06-07 22:09:58.000000000 -0500 +++ c/tools/python/xen/xm/main.py 2005-06-08 15:10:18.268757315 -0500 @@ -403,10 +403,11 @@ % d) def show_vcpus(self, doms): - print ''Name Id VCPU CPU CPUMAP'' + print ''Name Id VCPU CPU ONLINE CPUMAP'' for dom in doms: info = server.xend_domain(dom) vcpu_to_cpu = sxp.child_value(info, ''vcpu_to_cpu'', ''?'').replace(''-'','''') + vcpu_online_map = sxp.child_value(info, ''vcpu_online_map'', ''?'') cpumap = sxp.child_value(info, ''cpumap'', []) mask = ((int(sxp.child_value(info, ''vcpus'', ''0'')))**2) - 1 count = 0 @@ -416,9 +417,10 @@ d[''dom''] = int(sxp.child_value(info, ''id'', ''-1'')) d[''vcpu''] = int(count) d[''cpu''] = int(cpu) + d[''online''] = int(vcpu_online_map[count]) d[''cpumap''] = int(cpumap[count])&mask count = count + 1 - print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d 0x%(cpumap)x" % d) + print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d %(online)6d 0x%(cpumap)x" % d) def long_list(self, doms): for dom in doms: diff -urN b/xen/common/dom0_ops.c c/xen/common/dom0_ops.c --- b/xen/common/dom0_ops.c 2005-06-07 22:09:59.000000000 -0500 +++ c/xen/common/dom0_ops.c 2005-06-08 14:09:38.000000000 -0500 @@ -304,6 +304,7 @@ struct vcpu *v; u64 cpu_time = 0; int vcpu_count = 0; + u32 vcpu_online_map = 0; int flags = DOMFLAGS_PAUSED | DOMFLAGS_BLOCKED; read_lock(&domlist_lock); @@ -346,11 +347,15 @@ flags |= DOMFLAGS_RUNNING; if ( v->cpu_time > cpu_time ) cpu_time += v->cpu_time; + /* mark which vcpus are not down */ + if ( !(test_bit(_VCPUF_down, &v->vcpu_flags)) ) + vcpu_online_map |= (1<<v->vcpu_id); vcpu_count++; } op->u.getdomaininfo.cpu_time = cpu_time; op->u.getdomaininfo.n_vcpu = vcpu_count; + op->u.getdomaininfo.vcpu_online_map = vcpu_online_map; op->u.getdomaininfo.flags = flags | ((d->domain_flags & DOMF_dying) ? DOMFLAGS_DYING : 0) | @@ -373,7 +378,8 @@ { struct vcpu_guest_context *c; struct domain *d; - struct vcpu *v; + struct vcpu *v = NULL; + int i; d = find_domain_by_id(op->u.getvcpucontext.domain); if ( d == NULL ) @@ -388,8 +394,15 @@ put_domain(d); break; } + + /* find first valid vcpu starting from request. */ + for ( i=op->u.getvcpucontext.vcpu; i<MAX_VIRT_CPUS; i++ ) + { + v = d->vcpu[i]; + if ( v != NULL && !(test_bit(_VCPUF_down, &v->vcpu_flags)) ) + break; + } - v = d->vcpu[op->u.getvcpucontext.vcpu]; if ( v == NULL ) { ret = -ESRCH; diff -urN b/xen/common/schedule.c c/xen/common/schedule.c --- b/xen/common/schedule.c 2005-06-07 22:10:02.000000000 -0500 +++ c/xen/common/schedule.c 2005-06-08 12:57:58.000000000 -0500 @@ -261,6 +261,44 @@ return 0; } +/* Mark target vcpu as non-runnable so it is not scheduled */ +static long do_vcpu_down(int vcpu) +{ + struct vcpu *target; + + if (vcpu > MAX_VIRT_CPUS) + return -EINVAL; + + target = current->domain->vcpu[vcpu]; + /* DEBUG + * printk("DOM%d VCPU%d going down\n", + * target->domain->domain_id, target->vcpu_id); + */ + set_bit(_VCPUF_down, &target->vcpu_flags); + + return 0; +} + +/* Mark target vcpu as runnable and wake it */ +static long do_vcpu_up(int vcpu) +{ + struct vcpu *target; + + if (vcpu > MAX_VIRT_CPUS) + return -EINVAL; + + target = current->domain->vcpu[vcpu]; + /* DEBUG + * printk("DOM%d VCPU%d coming up\n", + * target->domain->domain_id, target->vcpu_id); + */ + clear_bit(_VCPUF_down, &target->vcpu_flags); + /* wake vcpu */ + domain_wake(target); + + return 0; +} + /* * Demultiplex scheduler-related hypercalls. */ @@ -290,6 +328,16 @@ domain_shutdown((u8)(op >> SCHEDOP_reasonshift)); break; } + case SCHEDOP_vcpu_down: + { + ret = do_vcpu_down((int)(op >> SCHEDOP_vcpushift)); + break; + } + case SCHEDOP_vcpu_up: + { + ret = do_vcpu_up((int)(op >> SCHEDOP_vcpushift)); + break; + } default: ret = -ENOSYS; diff -urN b/xen/include/public/dom0_ops.h c/xen/include/public/dom0_ops.h --- b/xen/include/public/dom0_ops.h 2005-06-07 22:10:00.000000000 -0500 +++ c/xen/include/public/dom0_ops.h 2005-06-08 12:57:58.728731646 -0500 @@ -85,7 +85,8 @@ memory_t max_pages; memory_t shared_info_frame; /* MFN of shared_info struct */ u64 cpu_time; - u32 n_vcpu; + u32 n_vcpu; /* number of vcpus allocated */ + u32 vcpu_online_map; /* bitmap of online vcpus */ s32 vcpu_to_cpu[MAX_VIRT_CPUS]; /* current mapping */ cpumap_t cpumap[MAX_VIRT_CPUS]; /* allowable mapping */ } dom0_getdomaininfo_t; diff -urN b/xen/include/public/xen.h c/xen/include/public/xen.h --- b/xen/include/public/xen.h 2005-06-07 22:10:00.000000000 -0500 +++ c/xen/include/public/xen.h 2005-06-08 12:57:58.729731508 -0500 @@ -200,8 +200,11 @@ #define SCHEDOP_yield 0 /* Give up the CPU voluntarily. */ #define SCHEDOP_block 1 /* Block until an event is received. */ #define SCHEDOP_shutdown 2 /* Stop executing this domain. */ +#define SCHEDOP_vcpu_down 3 /* make target VCPU not-runnable. */ +#define SCHEDOP_vcpu_up 4 /* make target VCPU runnable. */ #define SCHEDOP_cmdmask 255 /* 8-bit command. */ #define SCHEDOP_reasonshift 8 /* 8-bit reason code. (SCHEDOP_shutdown) */ +#define SCHEDOP_vcpushift 8 /* 8-bit VCPU target. (SCHEDOP_up|down) */ /* * Reason codes for SCHEDOP_shutdown. These may be interpreted by control diff -urN b/xen/include/xen/sched.h c/xen/include/xen/sched.h --- b/xen/include/xen/sched.h 2005-06-07 22:09:54.000000000 -0500 +++ c/xen/include/xen/sched.h 2005-06-08 12:57:58.000000000 -0500 @@ -346,6 +346,9 @@ /* Initialization completed. */ #define _VCPUF_initialised 8 #define VCPUF_initialised (1UL<<_VCPUF_initialised) + /* VCPU is not-runnable */ +#define _VCPUF_down 9 +#define VCPUF_down (1UL<<_VCPUF_down) /* * Per-domain flags (domain_flags). @@ -375,7 +378,7 @@ static inline int domain_runnable(struct vcpu *v) { return ( (atomic_read(&v->pausecnt) == 0) && - !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) && + !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) && !(v->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) ); } _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
Ryan Harper
2005-Jun-14 15:58 UTC
[Xen-devel] [PATCH][3rd try] don''t schedule unplugged vcpus
* Ryan Harper <ryanh@us.ibm.com> [2005-06-08 16:29]:> * Ryan Harper <ryanh@us.ibm.com> [2005-06-06 16:04]: > > This patch extends the CONFIG_HOTPLUG_CPU behavior down into the > > hypervisor. Currently when a CPU in Linux is moved offline, > > > > echo 0 > /sys/devices/system/cpu/cpuX/online > > > > the offline cpu yields its slice back to the hypervisor. This patch > > adds two SCHEDOPS (vcpu_down/vcpu_up) which set/clear a new VCPU flag, > > VCPU_down. The domain_runnable() check now looks at this flag and > > subsequently the vcpu is not scheduled when VCPU_down is set. > > > > The patch was built and tested against 20050606 nightly snapshot. > > Testing requires DOMU with CONFIG_SMP and CONFIG_HOTPLUG_CPU. Please > > apply. > > I''ve added in changes to DOM0_GETDOMINFO and DOM0_GETVCPUCONTEXT > hypercalls. dominfo now creates a vcpu_online_map bitmap which marks > whether vcpus are up or down. I didn''t want to clobber either the > total number of vcpus allocated to a domain (n_vcpu) nor the vcpu_to_cpu > mapping since both are still valid whether the vcpu is being scheduled > or not.Down VCPUS are accounted for in the vcpu_to_cpu map instead of a separate online map.> > I modified vcpucontext to give the context for the first vcpu not down. > If the requested vcpu is down, it will return the context of the next > vcpu that is up, or -ESRCH if no vcpu past the requested vcpu is valid.This has been kept.> I modified xm list -v to display an ONLINE column which indicates > the online status of each vcpu in a domain.I''ve removed the ONLINE column, using -1 in the CPU column to indicate offline VCPUS. Built and tested on 20050614 nightly unstable snapshot. Please apply. -- Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 ryanh@us.ibm.com diffstat output: linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c | 7 + linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 4 linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h | 31 +++++++ tools/python/xen/xm/main.py | 7 + xen/common/dom0_ops.c | 19 +++- xen/common/schedule.c | 48 +++++++++++ xen/include/public/xen.h | 3 xen/include/xen/sched.h | 5 - 8 files changed, 117 insertions(+), 7 deletions(-) Signed-off-by: Ryan Harper <ryanh@us.ibm.com> --- diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c --- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-12 22:13:37.000000000 -0500 +++ vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c 2005-06-13 10:41:56.000000000 -0500 @@ -154,8 +154,13 @@ cpu_clear(cpu, cpu_idle_map); rmb(); - if (cpu_is_offline(cpu)) + if (cpu_is_offline(cpu)) { +#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU) + /* Tell hypervisor not to schedule dead vcpus */ + HYPERVISOR_vcpu_down(cpu); +#endif play_dead(); + } irq_stat[cpu].idle_timestamp = jiffies; xen_idle(); diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c --- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-12 22:13:44.000000000 -0500 +++ vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-13 10:41:56.000000000 -0500 @@ -1380,6 +1380,10 @@ } #ifdef CONFIG_HOTPLUG_CPU +#ifdef CONFIG_XEN + /* Tell hypervisor to bring vcpu up */ + HYPERVISOR_vcpu_up(cpu); +#endif /* Already up, and in cpu_quiescent now? */ if (cpu_isset(cpu, smp_commenced_mask)) { cpu_enable(cpu); diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h vcpu_down/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h --- b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-12 22:13:45.000000000 -0500 +++ vcpu_down/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h 2005-06-13 10:41:56.000000000 -0500 @@ -517,4 +517,35 @@ return ret; } +static inline int +HYPERVISOR_vcpu_down( + int vcpu) +{ + int ret; + unsigned long ign1; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret), "=b" (ign1) + : "0" (__HYPERVISOR_sched_op), + "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift)) + : "memory" ); + + return ret; +} + +static inline int +HYPERVISOR_vcpu_up( + int vcpu) +{ + int ret; + unsigned long ign1; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret), "=b" (ign1) + : "0" (__HYPERVISOR_sched_op), + "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift)) + : "memory" ); + + return ret; +} #endif /* __HYPERCALL_H__ */ diff -urN b/tools/python/xen/xm/main.py vcpu_down/tools/python/xen/xm/main.py --- b/tools/python/xen/xm/main.py 2005-06-12 22:13:41.000000000 -0500 +++ vcpu_down/tools/python/xen/xm/main.py 2005-06-13 14:46:16.788968002 -0500 @@ -406,7 +406,7 @@ print ''Name Id VCPU CPU CPUMAP'' for dom in doms: info = server.xend_domain(dom) - vcpu_to_cpu = sxp.child_value(info, ''vcpu_to_cpu'', ''?'').replace(''-'','''') + vcpu_to_cpu = sxp.child_value(info, ''vcpu_to_cpu'', ''-1'').replace(''-1'',''#'') cpumap = sxp.child_value(info, ''cpumap'', []) mask = ((int(sxp.child_value(info, ''vcpus'', ''0'')))**2) - 1 count = 0 @@ -415,7 +415,10 @@ d[''name''] = sxp.child_value(info, ''name'', ''??'') d[''dom''] = int(sxp.child_value(info, ''id'', ''-1'')) d[''vcpu''] = int(count) - d[''cpu''] = int(cpu) + if cpu == "#": + d[''cpu''] = int("-1") + else: + d[''cpu''] = int(cpu) d[''cpumap''] = int(cpumap[count])&mask count = count + 1 print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d 0x%(cpumap)x" % d) diff -urN b/xen/common/dom0_ops.c vcpu_down/xen/common/dom0_ops.c --- b/xen/common/dom0_ops.c 2005-06-12 22:13:43.000000000 -0500 +++ vcpu_down/xen/common/dom0_ops.c 2005-06-13 11:09:35.000000000 -0500 @@ -334,9 +334,14 @@ * - domain is marked as paused or blocked only if all its vcpus * are paused or blocked * - domain is marked as running if any of its vcpus is running + * - only map vcpus that aren''t down. Note, at some point we may + * wish to demux the -1 value to indicate down vs. not-ever-booted + * */ for_each_vcpu ( d, v ) { - op->u.getdomaininfo.vcpu_to_cpu[v->vcpu_id] = v->processor; + /* only map vcpus that are up */ + if ( !(test_bit(_VCPUF_down, &v->vcpu_flags)) ) + op->u.getdomaininfo.vcpu_to_cpu[v->vcpu_id] = v->processor; op->u.getdomaininfo.cpumap[v->vcpu_id] = v->cpumap; if ( !(v->vcpu_flags & VCPUF_ctrl_pause) ) flags &= ~DOMFLAGS_PAUSED; @@ -373,7 +378,8 @@ { struct vcpu_guest_context *c; struct domain *d; - struct vcpu *v; + struct vcpu *v=NULL; + int i; d = find_domain_by_id(op->u.getvcpucontext.domain); if ( d == NULL ) @@ -388,8 +394,15 @@ put_domain(d); break; } + + /* find first valid vcpu starting from request. */ + for ( i=op->u.getvcpucontext.vcpu; i<MAX_VIRT_CPUS; i++ ) + { + v = d->vcpu[i]; + if ( v != NULL && !(test_bit(_VCPUF_down, &v->vcpu_flags)) ) + break; + } - v = d->vcpu[op->u.getvcpucontext.vcpu]; if ( v == NULL ) { ret = -ESRCH; diff -urN b/xen/common/schedule.c vcpu_down/xen/common/schedule.c --- b/xen/common/schedule.c 2005-06-12 22:13:45.000000000 -0500 +++ vcpu_down/xen/common/schedule.c 2005-06-13 10:41:57.000000000 -0500 @@ -261,6 +261,44 @@ return 0; } +/* Mark target vcpu as non-runnable so it is not scheduled */ +static long do_vcpu_down(int vcpu) +{ + struct vcpu *target; + + if (vcpu > MAX_VIRT_CPUS) + return -EINVAL; + + target = current->domain->vcpu[vcpu]; + /* DEBUG + * printk("DOM%d VCPU%d going down\n", + * target->domain->domain_id, target->vcpu_id); + */ + set_bit(_VCPUF_down, &target->vcpu_flags); + + return 0; +} + +/* Mark target vcpu as runnable and wake it */ +static long do_vcpu_up(int vcpu) +{ + struct vcpu *target; + + if (vcpu > MAX_VIRT_CPUS) + return -EINVAL; + + target = current->domain->vcpu[vcpu]; + /* DEBUG + * printk("DOM%d VCPU%d coming up\n", + * target->domain->domain_id, target->vcpu_id); + */ + clear_bit(_VCPUF_down, &target->vcpu_flags); + /* wake vcpu */ + domain_wake(target); + + return 0; +} + /* * Demultiplex scheduler-related hypercalls. */ @@ -290,6 +328,16 @@ domain_shutdown((u8)(op >> SCHEDOP_reasonshift)); break; } + case SCHEDOP_vcpu_down: + { + ret = do_vcpu_down((int)(op >> SCHEDOP_vcpushift)); + break; + } + case SCHEDOP_vcpu_up: + { + ret = do_vcpu_up((int)(op >> SCHEDOP_vcpushift)); + break; + } default: ret = -ENOSYS; diff -urN b/xen/include/public/xen.h vcpu_down/xen/include/public/xen.h --- b/xen/include/public/xen.h 2005-06-12 22:13:44.000000000 -0500 +++ vcpu_down/xen/include/public/xen.h 2005-06-13 10:41:57.000000000 -0500 @@ -200,8 +200,11 @@ #define SCHEDOP_yield 0 /* Give up the CPU voluntarily. */ #define SCHEDOP_block 1 /* Block until an event is received. */ #define SCHEDOP_shutdown 2 /* Stop executing this domain. */ +#define SCHEDOP_vcpu_down 3 /* make target VCPU not-runnable. */ +#define SCHEDOP_vcpu_up 4 /* make target VCPU runnable. */ #define SCHEDOP_cmdmask 255 /* 8-bit command. */ #define SCHEDOP_reasonshift 8 /* 8-bit reason code. (SCHEDOP_shutdown) */ +#define SCHEDOP_vcpushift 8 /* 8-bit VCPU target. (SCHEDOP_up|down) */ /* * Reason codes for SCHEDOP_shutdown. These may be interpreted by control diff -urN b/xen/include/xen/sched.h vcpu_down/xen/include/xen/sched.h --- b/xen/include/xen/sched.h 2005-06-12 22:13:38.000000000 -0500 +++ vcpu_down/xen/include/xen/sched.h 2005-06-13 10:41:57.000000000 -0500 @@ -346,6 +346,9 @@ /* Initialization completed. */ #define _VCPUF_initialised 8 #define VCPUF_initialised (1UL<<_VCPUF_initialised) + /* VCPU is not-runnable */ +#define _VCPUF_down 9 +#define VCPUF_down (1UL<<_VCPUF_down) /* * Per-domain flags (domain_flags). @@ -375,7 +378,7 @@ static inline int domain_runnable(struct vcpu *v) { return ( (atomic_read(&v->pausecnt) == 0) && - !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) && + !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) && !(v->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) ); } _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel