Ryan Grimm
2006-Feb-22 22:47 UTC
[Xen-devel] [PATCH] enforcing a dynamic vcpu limit for a domain
Hi Keir,
This is a followup to our previous discussion about enforcing a dynamic
vcpu limit for a domain*. I implemented an enforcement policy in xen
and distinguished max vcpus from initial vcpus in the config file.
CHANGES
1) xen - added DOM0_ALLOW_VCPU and VCPUOP_is_allowed.
Xen enforces a "hard limit" by a bit in each vcpu''s flags.
Xen will not
allow a vcpu_up op if the VCPUF_allowed bit is not set. The allow bit
is set by DOM0_ALLOW_VCPU, which accepts a 32-bit cpu map value.
2) sparse tree - smpboot.c calls VCPUOP_is_allowed before VCPUOP_up
3) tools - changes to vcpu-set, xend domain construction, and config file.
The value ''vcpus'' pertains to the maximum allowable vcpus and
''init_vcpus'' is the initial allowable cpus.
Vpu-set makes the DOM0_ALLOW_VCPU dom0_op, up to the max vcpus. The
domain''s hotplugging is still triggered by vcpu-set''s write to
the
store.
ISSUES:
The max is set via xend, and the default is 1. I suppose
the default could be something like 4, 8 or even MAX_VIRT_CPUS.
Also, lowering the allowable via vcpu-set triggers the domain''s hotplug
event to remove the cpus that are no longer allowed. One drawback of
this is the domain must be trusted to bring them down.
What do you think about this approach for enforcing a dynamic vcpu limit
for a domain?
Thanks,
Ryan
*http://lists.xensource.com/archives/html/xen-devel/2006-02/msg00561.html
Signed-off-by: Ryan Grimm <grimm@us.ibm.com>
diff -r 13e9fdaeed27 -r 7cd64c9dc532
linux-2.6-xen-sparse/drivers/xen/core/smpboot.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c Wed Feb 22 08:54:20 2006
+++ b/linux-2.6-xen-sparse/drivers/xen/core/smpboot.c Wed Feb 22 16:09:47 2006
@@ -419,6 +419,11 @@
prepare_for_smp();
#endif
+ if (!HYPERVISOR_vcpu_op(VCPUOP_is_allowed, cpu, NULL)) {
+ printk("cpu %d not allowed\n", cpu);
+ return -1;
+ }
+
xen_smp_intr_init(cpu);
cpu_set(cpu, cpu_online_map);
if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL) != 0)
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/examples/xmexample1
--- a/tools/examples/xmexample1 Wed Feb 22 08:54:20 2006
+++ b/tools/examples/xmexample1 Wed Feb 22 16:09:47 2006
@@ -27,8 +27,11 @@
#cpus = "0" # all vcpus run on CPU0
#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
-# Number of Virtual CPUS to use, default is 1
+# Maximum # of vcpus a comain can have in its life, default is 1
#vcpus = 1
+
+# Initial vcpus a domain is allowed to bring up, default is 1
+#init_vcpus = 1
#----------------------------------------------------------------------------
# Define network interfaces.
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/examples/xmexample2
--- a/tools/examples/xmexample2 Wed Feb 22 08:54:20 2006
+++ b/tools/examples/xmexample2 Wed Feb 22 16:09:47 2006
@@ -57,9 +57,12 @@
#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
#cpus = "%s" % vmid # set based on vmid (mod number of CPUs)
-# Number of Virtual CPUS to use, default is 1
+# Maximum # of vcpus a comain can have in its life, default is 1
#vcpus = 1
vcpus = 4 # make your domain a 4-way
+
+# Initial vcpus a domain is allowed to bring up, default is 1
+init_vcpus = 4 # allow the domain to bring up all 4 on boot
#----------------------------------------------------------------------------
# Define network interfaces.
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/libxc/xc_domain.c
--- a/tools/libxc/xc_domain.c Wed Feb 22 08:54:20 2006
+++ b/tools/libxc/xc_domain.c Wed Feb 22 16:09:47 2006
@@ -382,6 +382,15 @@
return do_dom0_op(xc_handle, &op);
}
+int xc_domain_allow_vcpus(int xc_handle, uint32_t domid, unsigned int allow)
+{
+ DECLARE_DOM0_OP;
+ op.cmd = DOM0_ALLOW_VCPUS;
+ op.u.allow_vcpus.domain = (domid_t)domid;
+ op.u.allow_vcpus.allow = allow;
+ return do_dom0_op(xc_handle, &op);
+}
+
int xc_domain_sethandle(int xc_handle, uint32_t domid,
xen_domain_handle_t handle)
{
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h Wed Feb 22 08:54:20 2006
+++ b/tools/libxc/xenctrl.h Wed Feb 22 16:09:47 2006
@@ -148,6 +148,18 @@
uint32_t domid,
unsigned int max);
+/*
+ * This function sets the vcpus that a domain may bring up
+ *
+ * @parm xc_handle a handle to an open hypervisor interface.
+ * @parm domid the domain id in which vcpus are to be created.
+ * @parm allow a bitmap of the vcpus that the domain may bringup.
+ * @return 0 on success, -1 on failure.
+ */
+int xc_domain_allow_vcpus(int xc_handle,
+ uint32_t domid,
+ unsigned int allow);
+
/**
* This function pauses a domain. A paused domain still exists in memory
* however it does not receive any timeslices from the hypervisor.
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c Wed Feb 22 08:54:20 2006
+++ b/tools/python/xen/lowlevel/xc/xc.c Wed Feb 22 16:09:47 2006
@@ -115,6 +115,20 @@
if (xc_domain_max_vcpus(self->xc_handle, dom, max) != 0)
return PyErr_SetFromErrno(xc_error);
+ Py_INCREF(zero);
+ return zero;
+}
+
+static PyObject *pyxc_domain_allow_vcpus(XcObject *self, PyObject *args)
+{
+ uint32_t dom, allowed;
+
+ if (!PyArg_ParseTuple(args, "ii", &dom, &allowed))
+ return NULL;
+
+ if (xc_domain_allow_vcpus(self->xc_handle, dom, allowed) != 0)
+ return PyErr_SetFromErrno(xc_error);
+
Py_INCREF(zero);
return zero;
}
@@ -861,6 +875,14 @@
"Set the maximum number of VCPUs a domain may create.\n"
" dom [int, 0]: Domain identifier to use.\n"
" max [int, 0]: New maximum number of VCPUs in
domain.\n"
+ "Returns: [int] 0 on success; -1 on error.\n" },
+
+ { "domain_allow_vcpus",
+ (PyCFunction)pyxc_domain_allow_vcpus,
+ METH_VARARGS, "\n"
+ "Set the allowable VCPUs a domain may use.\n"
+ " dom [int, 0]: Domain identifier to use.\n"
+ " allow [int, 0]: Bitmap of allowable CPUs in
domain.\n"
"Returns: [int] 0 on success; -1 on error.\n" },
{ "domain_dumpcore",
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py Wed Feb 22 08:54:20 2006
+++ b/tools/python/xen/xend/XendDomainInfo.py Wed Feb 22 16:09:47 2006
@@ -121,6 +121,7 @@
(''uuid'', str),
(''ssidref'', int),
(''vcpus'', int),
+ (''init_vcpus'', int),
(''vcpu_avail'', int),
(''cpu_weight'', float),
(''memory'', int),
@@ -553,7 +554,7 @@
defaultInfo(''vcpus'', lambda: avail)
defaultInfo(''online_vcpus'', lambda:
self.info[''vcpus''])
defaultInfo(''max_vcpu_id'', lambda:
self.info[''vcpus'']-1)
- defaultInfo(''vcpu_avail'', lambda: (1 <<
self.info[''vcpus'']) - 1)
+ defaultInfo(''vcpu_avail'', lambda: (1 <<
self.info[''init_vcpus'']) - 1)
defaultInfo(''memory'', lambda: 0)
defaultInfo(''maxmem'', lambda: 0)
@@ -750,6 +751,10 @@
def setVCpuCount(self, vcpus):
+ allow=0
+ for j in range(0,vcpus):
+ allow |= 1<<j
+ xc.domain_allow_vcpus(self.domid, allow)
self.info[''vcpu_avail''] = (1 << vcpus) - 1
self.storeVm(''vcpu_avail'',
self.info[''vcpu_avail''])
self.writeDom(self.vcpuDomDetails())
@@ -1154,8 +1159,13 @@
self.recreateDom()
# Set maximum number of vcpus in domain
- xc.domain_max_vcpus(self.domid,
int(self.info[''vcpus'']))
-
+ xc.domain_max_vcpus(self.domid,
int(self.info[''vcpus'']))
+
+ # Set allowable vcpus
+ allowable = 0
+ for i in range(0,int(self.info[''init_vcpus''])):
+ allowable |= (1 << i)
+ xc.domain_allow_vcpus(self.domid, allowable)
def introduceDomain(self):
assert self.domid is not None
diff -r 13e9fdaeed27 -r 7cd64c9dc532 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py Wed Feb 22 08:54:20 2006
+++ b/tools/python/xen/xm/create.py Wed Feb 22 16:09:47 2006
@@ -176,6 +176,10 @@
gopts.var(''vcpus'', val=''VCPUS'',
fn=set_int, default=1,
use="# of Virtual CPUS in domain.")
+
+gopts.var(''init_vcpus'', val=''INIT_VCPUS'',
+ fn=set_int, default=1,
+ use="initial vcpus in domain")
gopts.var(''cpu_weight'', val=''WEIGHT'',
fn=set_float, default=None,
@@ -581,7 +585,8 @@
config.append([n, v])
map(add_conf, [''name'', ''memory'',
''ssidref'', ''maxmem'',
''restart'',
- ''on_poweroff'',
''on_reboot'', ''on_crash'',
''vcpus''])
+ ''on_poweroff'',
''on_reboot'', ''on_crash'',
''vcpus'',
+ ''init_vcpus''])
if vals.uuid is not None:
config.append([''uuid'', vals.uuid])
diff -r 13e9fdaeed27 -r 7cd64c9dc532 xen/common/dom0_ops.c
--- a/xen/common/dom0_ops.c Wed Feb 22 08:54:20 2006
+++ b/xen/common/dom0_ops.c Wed Feb 22 16:09:47 2006
@@ -268,6 +268,24 @@
}
break;
+ case DOM0_ALLOW_VCPUS:
+ {
+ struct domain *d;
+ unsigned int allow = op->u.allow_vcpus.allow, i;
+
+ ret = -ESRCH;
+ if ( (d = find_domain_by_id(op->u.allow_vcpus.domain)) == NULL )
+ break;
+
+ for ( i = 0; i < MAX_VIRT_CPUS; i++ )
+ if (allow & 1UL<<i && d->vcpu[i] != NULL)
+ set_bit(_VCPUF_allowed, &d->vcpu[i]->vcpu_flags);
+
+ ret = 0;
+ put_domain(d);
+ }
+ break;
+
case DOM0_DESTROYDOMAIN:
{
struct domain *d = find_domain_by_id(op->u.destroydomain.domain);
diff -r 13e9fdaeed27 -r 7cd64c9dc532 xen/common/domain.c
--- a/xen/common/domain.c Wed Feb 22 08:54:20 2006
+++ b/xen/common/domain.c Wed Feb 22 16:09:47 2006
@@ -437,7 +437,9 @@
break;
case VCPUOP_up:
- if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
+ if ( !test_bit(_VCPUF_allowed, &v->vcpu_flags) )
+ rc = -EINVAL;
+ else if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
rc = -EINVAL;
else if ( test_and_clear_bit(_VCPUF_down, &v->vcpu_flags) )
vcpu_wake(v);
@@ -450,6 +452,10 @@
case VCPUOP_is_up:
rc = !test_bit(_VCPUF_down, &v->vcpu_flags);
+ break;
+
+ case VCPUOP_is_allowed:
+ rc = test_bit(_VCPUF_allowed, &v->vcpu_flags);
break;
}
diff -r 13e9fdaeed27 -r 7cd64c9dc532 xen/include/public/dom0_ops.h
--- a/xen/include/public/dom0_ops.h Wed Feb 22 08:54:20 2006
+++ b/xen/include/public/dom0_ops.h Wed Feb 22 16:09:47 2006
@@ -396,6 +396,12 @@
domid_t domain; /* domain to be affected */
uint32_t max; /* maximum number of vcpus */
} dom0_max_vcpus_t;
+
+#define DOM0_ALLOW_VCPUS 42
+typedef struct dom0_allow_vcpus {
+ domid_t domain;
+ uint32_t allow;
+} dom0_allow_vcpus_t;
#define DOM0_SETDOMAINHANDLE 44
typedef struct dom0_setdomainhandle {
@@ -466,6 +472,7 @@
struct dom0_platform_quirk platform_quirk;
struct dom0_physical_memory_map physical_memory_map;
struct dom0_max_vcpus max_vcpus;
+ struct dom0_allow_vcpus allow_vcpus;
struct dom0_setdomainhandle setdomainhandle;
struct dom0_setdebugging setdebugging;
struct dom0_irq_permission irq_permission;
diff -r 13e9fdaeed27 -r 7cd64c9dc532 xen/include/public/vcpu.h
--- a/xen/include/public/vcpu.h Wed Feb 22 08:54:20 2006
+++ b/xen/include/public/vcpu.h Wed Feb 22 16:09:47 2006
@@ -51,6 +51,9 @@
/* Returns 1 if the given VCPU is up. */
#define VCPUOP_is_up 3
+/* Returns 1 if the VCPU is allowed to brought up by the domain */
+#define VCPUOP_is_allowed 4
+
#endif /* __XEN_PUBLIC_VCPU_H__ */
/*
diff -r 13e9fdaeed27 -r 7cd64c9dc532 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h Wed Feb 22 08:54:20 2006
+++ b/xen/include/xen/sched.h Wed Feb 22 16:09:47 2006
@@ -377,6 +377,9 @@
/* Avoid NMI reentry by allowing NMIs to be masked for short periods. */
#define _VCPUF_nmi_masked 9
#define VCPUF_nmi_masked (1UL<<_VCPUF_nmi_masked)
+ /* Is this VCPU allowed to be brought up by the domain */
+#define _VCPUF_allowed 10
+#define VCPUF_allowed (1UL<<_VCPUF_allowed)
/*
* Per-domain flags (domain_flags).
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel