The ''cpu'' option in domain config files will pin VCPU0 of a
domain.
This is not as useful now that domains can have more than 1 vcpu. This
patch changes ''cpu'' to ''cpus'' and takes a
list of physical cpus the
domains'' vcpus can use and will pin the vcpus upon domain creation.
cpus = [1] # this starts all domain vcpus pinned to CPU1
The list is circular, so in a domain with the following config:
vcpus = 4
cpus = [0,3,7] # Use any of 0, 3, 7 for this domain.
would see vcpus 0-3 pinned to cpus 0,3,7,0 respectively.
Also, the pin operation is moved before the memory reservation as vcpu
to cpu mapping will be helpful for future NUMA work when trying to
allocate pages close to the physical cpus being used.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
examples/xmexample.vmx | 5 +++--
examples/xmexample.vti | 5 +++--
examples/xmexample1 | 5 +++--
examples/xmexample2 | 7 ++++---
examples/xmexample3 | 7 ++++---
python/xen/xend/XendDomainInfo.py | 18 ++++++++++++------
python/xen/xm/create.py | 8 ++++----
7 files changed, 33 insertions(+), 22 deletions(-)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -r f970d1ad3234 tools/examples/xmexample.vmx
--- a/tools/examples/xmexample.vmx Wed Nov 16 16:44:52 2005
+++ b/tools/examples/xmexample.vmx Wed Nov 16 14:39:21 2005
@@ -30,8 +30,9 @@
# the number of cpus guest platform has, default=1
vcpus=1
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = [] # leave to Xen to pick
+#cpus = [0] # all vcpus run on CPU0
# Optionally define mac and/or bridge for the network interfaces.
# Random MACs are assigned if not given.
diff -r f970d1ad3234 tools/examples/xmexample.vti
--- a/tools/examples/xmexample.vti Wed Nov 16 16:44:52 2005
+++ b/tools/examples/xmexample.vti Wed Nov 16 14:39:21 2005
@@ -23,8 +23,9 @@
# A name for your domain. All domains must have different names.
name = "ExampleVMXDomain"
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = [] # leave to Xen to pick
+#cpus = [0] # all vcpus run on CPU0
# Disable vif for now
nics=0
diff -r f970d1ad3234 tools/examples/xmexample1
--- a/tools/examples/xmexample1 Wed Nov 16 16:44:52 2005
+++ b/tools/examples/xmexample1 Wed Nov 16 14:39:21 2005
@@ -22,8 +22,9 @@
# A name for your domain. All domains must have different names.
name = "ExampleDomain"
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = [] # leave to Xen to pick
+#cpus = [0] # all vcpus run on CPU0
# Number of Virtual CPUS to use, default is 1
#vcpus = 1
diff -r f970d1ad3234 tools/examples/xmexample2
--- a/tools/examples/xmexample2 Wed Nov 16 16:44:52 2005
+++ b/tools/examples/xmexample2 Wed Nov 16 14:39:21 2005
@@ -51,9 +51,10 @@
# so we use the vmid to create a name.
name = "VM%d" % vmid
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
-cpu = vmid # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = [] # leave to Xen to pick
+#cpus = [0] # all vcpus run on CPU0
+cpus = [vmid] # set based on vmid (mod number of CPUs)
# Number of Virtual CPUS to use, default is 1
#vcpus = 1
diff -r f970d1ad3234 tools/examples/xmexample3
--- a/tools/examples/xmexample3 Wed Nov 16 16:44:52 2005
+++ b/tools/examples/xmexample3 Wed Nov 16 14:39:21 2005
@@ -51,9 +51,10 @@
# so we use the vmid to create a name.
name = "VM%d" % vmid
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
-cpu = vmid # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = [] # leave to Xen to pick
+#cpus = [0] # all vcpus run on CPU0
+cpus = [vmid] # set based on vmid (mod number of CPUs)
#----------------------------------------------------------------------------
# Define network interfaces.
diff -r f970d1ad3234 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py Wed Nov 16 16:44:52 2005
+++ b/tools/python/xen/xend/XendDomainInfo.py Wed Nov 16 14:39:21 2005
@@ -258,7 +258,7 @@
for e in ROUNDTRIPPING_CONFIG_ENTRIES:
result[e[0]] = get_cfg(e[0], e[1])
- result[''cpu''] = get_cfg(''cpu'',
int)
+ result[''cpus''] = get_cfg(''cpus'',
list)
result[''image''] = get_cfg(''image'')
try:
@@ -435,7 +435,7 @@
defaultInfo(''on_poweroff'', lambda:
"destroy")
defaultInfo(''on_reboot'', lambda:
"restart")
defaultInfo(''on_crash'', lambda:
"restart")
- defaultInfo(''cpu'', lambda: None)
+ defaultInfo(''cpus'', lambda: [])
defaultInfo(''cpu_weight'', lambda: 1.0)
# some domains don''t have a config file (e.g. dom0 )
@@ -1056,13 +1056,19 @@
xc.domain_setcpuweight(self.domid,
self.info[''cpu_weight''])
+ # repin domain vcpus if a restricted cpus list is provided
+ # this is done prior to memory allocation to aide in memory
+ # distribution for NUMA systems.
+ cpus = self.info[''cpus'']
+ if cpus is not None and len(cpus) > 0:
+ for v in range(0, self.info[''max_vcpu_id'']+1):
+ # pincpu takes a list of ints
+ cpu = [ int( cpus[v % len(cpus)] ) ]
+ xc.domain_pincpu(self.domid, v, cpu)
+
m = self.image.getDomainMemory(self.info[''memory''] *
1024)
xc.domain_setmaxmem(self.domid, maxmem_kb = m)
xc.domain_memory_increase_reservation(self.domid, m, 0, 0)
-
- cpu = self.info[''cpu'']
- if cpu is not None and cpu != -1:
- xc.domain_pincpu(self.domid, 0, 1 << cpu)
self.createChannels()
diff -r f970d1ad3234 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py Wed Nov 16 16:44:52 2005
+++ b/tools/python/xen/xm/create.py Wed Nov 16 14:39:21 2005
@@ -152,9 +152,9 @@
fn=set_int, default=None,
use="Maximum domain memory in MB.")
-gopts.var(''cpu'', val=''CPU'',
+gopts.var(''cpus'', val=''CPUS'',
fn=set_int, default=None,
- use="CPU to run the domain on.")
+ use="CPUS to run the domain on.")
gopts.var(''lapic'', val=''LAPIC'',
fn=set_int, default=0,
@@ -582,8 +582,8 @@
map(add_conf, [''name'', ''memory'',
''ssidref'', ''maxmem'',
''restart'',
''on_poweroff'',
''on_reboot'', ''on_crash''])
- if vals.cpu is not None:
- config.append([''cpu'', vals.cpu])
+ if vals.cpus is not None:
+ config.append([''cpus'', vals.cpus])
if vals.cpu_weight is not None:
config.append([''cpu_weight'', vals.cpu_weight])
if vals.blkif:
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel
> The ''cpu'' option in domain config files will pin VCPU0 of a domain. > This is not as useful now that domains can have more than 1 > vcpu. This patch changes ''cpu'' to ''cpus'' and takes a list of > physical cpus the domains'' vcpus can use and will pin the > vcpus upon domain creation. > > cpus = [1] # this starts all domain vcpus pinned to CPU1I think this patch is generally a good thing. Should we support cpu= as backward compatible legacy option?> The list is circular, so in a domain with the following config: > > vcpus = 4 > cpus = [0,3,7] # Use any of 0, 3, 7 for this domain. > > would see vcpus 0-3 pinned to cpus 0,3,7,0 respectively.Actually, although this is a reasonable syntax, I think we''ll probably interpret it differently in future when we have CPU load ballancing: I think we''ll want to list the set of CPUs that a given _domain_ can use rather than pining individual CPUs. However, I wander whether this should be a string so that we can list e.g. cpus=''0-3,5,^1'' Ian _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
* Ian Pratt <m+Ian.Pratt@cl.cam.ac.uk> [2005-11-16 16:07]:> > The ''cpu'' option in domain config files will pin VCPU0 of a domain. > > This is not as useful now that domains can have more than 1 > > vcpu. This patch changes ''cpu'' to ''cpus'' and takes a list of > > physical cpus the domains'' vcpus can use and will pin the > > vcpus upon domain creation. > > > > cpus = [1] # this starts all domain vcpus pinned to CPU1 > > I think this patch is generally a good thing. Should we support cpu= as > backward compatible legacy option?I pondered that as well. No reason we can''t. Some documentation should avoid any confusion. I''ll rework the patch to support cpu=X and cpus=[].> > > The list is circular, so in a domain with the following config: > > > > vcpus = 4 > > cpus = [0,3,7] # Use any of 0, 3, 7 for this domain. > > > > would see vcpus 0-3 pinned to cpus 0,3,7,0 respectively. > > Actually, although this is a reasonable syntax, I think we''ll probably > interpret it differently in future when we have CPU load ballancing: I > think we''ll want to list the set of CPUs that a given _domain_ can use > rather than pining individual CPUs.I think the list is already representative of that idea: this is a list of cpus that any of the vcpus in this domain can use. Currently without a load balancer we only get one go at vcpu to cpu allocation. Also, no reason we can''t replace the current algorithm down the road with a call out to the load balancer which would supply the mappings.> However, I wander whether this should be a string so that we can list > e.g. cpus=''0-3,5,^1''I like that notation better, but it opens up a few questions. Do you mean the commas to indicate which cpus each vcpu is allowed to use, or just a list of cpus the domain can use? Also, I take ^1 to mean any cpu, yes? I''ll end up converting the string into a big list of possible cpus to use and for each vcpu pick a cpu and pin it there. -- Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 ryanh@us.ibm.com _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
> I think the list is already representative of that idea: this > is a list of cpus that any of the vcpus in this domain can > use. Currently without a load balancer we only get one go at > vcpu to cpu allocation. > > Also, no reason we can''t replace the current algorithm down > the road with a call out to the load balancer which would > supply the mappings.Yes, that''s what I was getting at.> > However, I wander whether this should be a string so that > we can list > > e.g. cpus=''0-3,5,^1'' > > I like that notation better, but it opens up a few questions. > Do you mean the commas to indicate which cpus each vcpu is > allowed to use, or just a list of cpus the domain can use?Just a list the domain can use.> Also, I take ^1 to mean any cpu, yes?No, I meant ''not CPU 1''. It takes precendence over 0-3 as its later in the list, so the final set of usable CPUs for this domain would be 0,2,3,5> I''ll end up converting the string into a big list of possible > cpus to use and for each vcpu pick a cpu and pin it there.That''s good for the moment, though we''ll probably turn it into a list of CPUs available to the domain. Best, Ian _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
Ryan Harper
2005-Nov-18 23:14 UTC
[Xen-devel] [PATCH][2nd try] xend: add cpus config option
This patch adds a new domain config option, ''cpus'' which is a
list of
CPUs a domains'' vcpus can use. The older ''cpu''
config option is
prepended to the list of cpus to use and will keep the behavior of
pinning VCPU0.
The cpus option supports ranges and negation, so:
cpus = "0-3,5,^1" produces -> [0,2,3,5]
The list is circular, so in a domain with the following config:
vcpus = 4
cpus = "0,3,7" # Use any of 0, 3, 7 for this domain.
would see vcpus 0-3 pinned to cpus 0,3,7,0 respectively.
Also, the pin operation is moved before the memory reservation as vcpu
to cpu mapping will be helpful for future NUMA work when trying to
allocate pages close to the physical cpus being used.
An update to the display of cpumap was needed to normalize the cpumap
values to the range of possible cpus.
I''ve also included some text for the xmdomain.cfg(5) man page.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
docs/man/xmdomain.cfg.pod.5 | 10 ++++++
tools/examples/xmexample.vmx | 6 ++-
tools/examples/xmexample.vti | 6 ++-
tools/examples/xmexample1 | 6 ++-
tools/examples/xmexample2 | 8 +++-
tools/examples/xmexample3 | 8 +++-
tools/python/xen/xend/XendDomainInfo.py | 53 +++++++++++++++++++++++++++++---
tools/python/xen/xm/create.py | 8 ++++
tools/python/xen/xm/main.py | 4 +-
9 files changed, 91 insertions(+), 18 deletions(-)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
---
diff -r 378e1c58bcd2 docs/man/xmdomain.cfg.pod.5
--- a/docs/man/xmdomain.cfg.pod.5 Fri Nov 18 16:54:23 2005
+++ b/docs/man/xmdomain.cfg.pod.5 Fri Nov 18 16:14:45 2005
@@ -156,6 +156,16 @@
the first cpu, 1 the second, and so on. This defaults to -1, which
means Xen is free to pick which CPU to start on.
+=item B<cpus>
+
+Specifies a list of CPUs on which the domains'' VCPUs are allowed to
+execute upon. The syntax supports ranges (0-3), and negation, ^1.
+For instance:
+
+ cpus = "0-3,5,^1"
+
+Will result in CPUs 0, 2, 3, 5 being available for use by the domain.
+
=item B<extra>
Extra information to append to the end of the kernel parameter line.
diff -r 378e1c58bcd2 tools/examples/xmexample.vmx
--- a/tools/examples/xmexample.vmx Fri Nov 18 16:54:23 2005
+++ b/tools/examples/xmexample.vmx Fri Nov 18 16:14:45 2005
@@ -30,8 +30,10 @@
# the number of cpus guest platform has, default=1
vcpus=1
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = "" # leave to Xen to pick
+#cpus = "0" # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
# Optionally define mac and/or bridge for the network interfaces.
# Random MACs are assigned if not given.
diff -r 378e1c58bcd2 tools/examples/xmexample.vti
--- a/tools/examples/xmexample.vti Fri Nov 18 16:54:23 2005
+++ b/tools/examples/xmexample.vti Fri Nov 18 16:14:45 2005
@@ -23,8 +23,10 @@
# A name for your domain. All domains must have different names.
name = "ExampleVMXDomain"
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = "" # leave to Xen to pick
+#cpus = "0" # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
# Disable vif for now
nics=0
diff -r 378e1c58bcd2 tools/examples/xmexample1
--- a/tools/examples/xmexample1 Fri Nov 18 16:54:23 2005
+++ b/tools/examples/xmexample1 Fri Nov 18 16:14:45 2005
@@ -22,8 +22,10 @@
# A name for your domain. All domains must have different names.
name = "ExampleDomain"
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = "" # leave to Xen to pick
+#cpus = "0" # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
# Number of Virtual CPUS to use, default is 1
#vcpus = 1
diff -r 378e1c58bcd2 tools/examples/xmexample2
--- a/tools/examples/xmexample2 Fri Nov 18 16:54:23 2005
+++ b/tools/examples/xmexample2 Fri Nov 18 16:14:45 2005
@@ -51,9 +51,11 @@
# so we use the vmid to create a name.
name = "VM%d" % vmid
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
-cpu = vmid # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = "" # leave to Xen to pick
+#cpus = "0" # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
+cpus = "%s" % vmid # set based on vmid (mod number of CPUs)
# Number of Virtual CPUS to use, default is 1
#vcpus = 1
diff -r 378e1c58bcd2 tools/examples/xmexample3
--- a/tools/examples/xmexample3 Fri Nov 18 16:54:23 2005
+++ b/tools/examples/xmexample3 Fri Nov 18 16:14:45 2005
@@ -51,9 +51,11 @@
# so we use the vmid to create a name.
name = "VM%d" % vmid
-# Which CPU to start domain on?
-#cpu = -1 # leave to Xen to pick
-cpu = vmid # set based on vmid (mod number of CPUs)
+# List of which CPUS this domain is allowed to use, default Xen picks
+#cpus = "" # leave to Xen to pick
+#cpus = "0" # all vcpus run on CPU0
+#cpus = "0-3,5,^1" # run on cpus 0,2,3,5
+cpus = "%s" % vmid # set based on vmid (mod number of CPUs)
#----------------------------------------------------------------------------
# Define network interfaces.
diff -r 378e1c58bcd2 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py Fri Nov 18 16:54:23 2005
+++ b/tools/python/xen/xend/XendDomainInfo.py Fri Nov 18 16:14:45 2005
@@ -288,6 +288,7 @@
result[e[0]] = get_cfg(e[0], e[1])
result[''cpu''] = get_cfg(''cpu'',
int)
+ result[''cpus''] = get_cfg(''cpus'',
str)
result[''image''] = get_cfg(''image'')
try:
@@ -300,6 +301,43 @@
raise VmError(
''Invalid configuration setting: vcpus = %s: %s'' %
(sxp.child_value(result[''image''],
''vcpus'', 1), str(exn)))
+
+ try:
+ # support legacy config files with ''cpu'' parameter
+ # NB: prepending to list to try support previous behavior
+ # where ''cpu'' parameter pinned VCPU0.
+ if result[''cpu'']:
+ if result[''cpus'']:
+ result[''cpus''] = "%s,%s" %
(str(result[''cpu'']), result[''cpus''])
+ else:
+ result[''cpus''] =
str(result[''cpu''])
+
+ # convert ''cpus'' string to list of ints
+ # ''cpus'' supports a list of ranges (0-3), seperated
by
+ # commas, and negation, (^1).
+ # Precedence is settled by order of the string:
+ # "0-3,^1" -> [0,2,3]
+ # "0-3,^1,1" -> [0,1,2,3]
+ if result[''cpus'']:
+ cpus = []
+ for c in
result[''cpus''].split('',''):
+ if c.find(''-'') != -1:
+ (x,y) = c.split(''-'')
+ for i in range(int(x),int(y)+1):
+ cpus.append(int(i))
+ else:
+ # remove this element from the list
+ if c[0] == ''^'':
+ cpus = [x for x in cpus if x != int(c[1])]
+ else:
+ cpus.append(int(c))
+
+ result[''cpus''] = cpus
+
+ except ValueError, exn:
+ raise VmError(
+ ''Invalid configuration setting: cpus = %s: %s'' %
+ (result[''cpus''], exn))
result[''backend''] = []
for c in sxp.children(config, ''backend''):
@@ -481,6 +519,7 @@
defaultInfo(''on_reboot'', lambda:
"restart")
defaultInfo(''on_crash'', lambda:
"restart")
defaultInfo(''cpu'', lambda: None)
+ defaultInfo(''cpus'', lambda: [])
defaultInfo(''cpu_weight'', lambda: 1.0)
# some domains don''t have a config file (e.g. dom0 )
@@ -1105,13 +1144,19 @@
xc.domain_setcpuweight(self.domid,
self.info[''cpu_weight''])
+ # repin domain vcpus if a restricted cpus list is provided
+ # this is done prior to memory allocation to aide in memory
+ # distribution for NUMA systems.
+ cpus = self.info[''cpus'']
+ if cpus is not None and len(cpus) > 0:
+ for v in range(0, self.info[''max_vcpu_id'']+1):
+ # pincpu takes a list of ints
+ cpu = [ int( cpus[v % len(cpus)] ) ]
+ xc.domain_pincpu(self.domid, v, cpu)
+
m = self.image.getDomainMemory(self.info[''memory''] *
1024)
xc.domain_setmaxmem(self.domid, maxmem_kb = m)
xc.domain_memory_increase_reservation(self.domid, m, 0, 0)
-
- cpu = self.info[''cpu'']
- if cpu is not None and cpu != -1:
- xc.domain_pincpu(self.domid, 0, 1 << cpu)
self.createChannels()
diff -r 378e1c58bcd2 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py Fri Nov 18 16:54:23 2005
+++ b/tools/python/xen/xm/create.py Fri Nov 18 16:14:45 2005
@@ -154,7 +154,11 @@
gopts.var(''cpu'', val=''CPU'',
fn=set_int, default=None,
- use="CPU to run the domain on.")
+ use="CPU to run the VCPU0 on.")
+
+gopts.var(''cpus'', val=''CPUS'',
+ fn=set_int, default=None,
+ use="CPUS to run the domain on.")
gopts.var(''lapic'', val=''LAPIC'',
fn=set_int, default=0,
@@ -584,6 +588,8 @@
if vals.cpu is not None:
config.append([''cpu'', vals.cpu])
+ if vals.cpus is not None:
+ config.append([''cpus'', vals.cpus])
if vals.cpu_weight is not None:
config.append([''cpu_weight'', vals.cpu_weight])
if vals.blkif:
diff -r 378e1c58bcd2 tools/python/xen/xm/main.py
--- a/tools/python/xen/xm/main.py Fri Nov 18 16:54:23 2005
+++ b/tools/python/xen/xm/main.py Fri Nov 18 16:14:45 2005
@@ -439,7 +439,9 @@
for x in server.xend_node()[1:]:
if len(x) > 1 and x[0] == ''nr_cpus'':
nr_cpus = int(x[1])
- cpumap = filter(lambda x: x < nr_cpus, cpumap)
+ # normalize cpumap by modulus nr_cpus, and drop duplicates
+ cpumap = dict.fromkeys(
+ map(lambda x: x % nr_cpus, cpumap)).keys()
if len(cpumap) == nr_cpus:
return "any cpu"
break
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel