Add nomigrate config option to disable migration/restore The new nomigrate option can be set to non-zero in vm.cfg (for both hvm and pvm) to disallow a guest from being migrated or restored. (Save is still allowed for the purpose of checkpointing.) The option persists into a save file and is also communicated into the hypervisor, the latter for the purposes of a to-be-added hypercall for communicating to guests that migration is disallowed (which will be used initially for userland TSC-related sensing, but may find other uses). Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com> diff -r ea34183c5c11 tools/libxc/xc_domain.c --- a/tools/libxc/xc_domain.c Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/libxc/xc_domain.c Mon Oct 19 15:50:06 2009 -0600 @@ -472,6 +472,15 @@ int xc_domain_set_tsc_native(int xc_hand domctl.cmd = XEN_DOMCTL_set_tsc_native; domctl.domain = (domid_t)domid; domctl.u.set_tsc_native.is_native = is_native; + return do_domctl(xc_handle, &domctl); +} + +int xc_domain_disable_migrate(int xc_handle, uint32_t domid) +{ + DECLARE_DOMCTL; + domctl.cmd = XEN_DOMCTL_disable_migrate; + domctl.domain = (domid_t)domid; + domctl.u.disable_migrate.disable = 1; return do_domctl(xc_handle, &domctl); } diff -r ea34183c5c11 tools/libxc/xenctrl.h --- a/tools/libxc/xenctrl.h Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/libxc/xenctrl.h Mon Oct 19 15:50:06 2009 -0600 @@ -629,6 +629,8 @@ int xc_domain_set_time_offset(int xc_han int32_t time_offset_seconds); int xc_domain_set_tsc_native(int xc_handle, uint32_t domid, int is_native); + +int xc_domain_disable_migrate(int xc_handle, uint32_t domid); int xc_domain_memory_increase_reservation(int xc_handle, uint32_t domid, diff -r ea34183c5c11 tools/python/xen/lowlevel/xc/xc.c --- a/tools/python/xen/lowlevel/xc/xc.c Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/lowlevel/xc/xc.c Mon Oct 19 15:50:06 2009 -0600 @@ -1479,6 +1479,20 @@ static PyObject *pyxc_domain_set_tsc_nat return NULL; if (xc_domain_set_tsc_native(self->xc_handle, dom, is_native) != 0) + return pyxc_error_to_exception(); + + Py_INCREF(zero); + return zero; +} + +static PyObject *pyxc_domain_disable_migrate(XcObject *self, PyObject *args) +{ + uint32_t dom; + + if (!PyArg_ParseTuple(args, "i", &dom)) + return NULL; + + if (xc_domain_disable_migrate(self->xc_handle, dom) != 0) return pyxc_error_to_exception(); Py_INCREF(zero); @@ -2015,6 +2029,13 @@ static PyMethodDef pyxc_methods[] = { " is_native [int]: 1=native, 0=emulate.\n" "Returns: [int] 0 on success; -1 on error.\n" }, + { "domain_disable_migrate", + (PyCFunction)pyxc_domain_disable_migrate, + METH_VARARGS, "\n" + "Marks domain as non-migratable AND non-restoreable\n" + " dom [int]: Domain whose TSC mode is being set.\n" + "Returns: [int] 0 on success; -1 on error.\n" }, + { "domain_send_trigger", (PyCFunction)pyxc_domain_send_trigger, METH_VARARGS | METH_KEYWORDS, "\n" diff -r ea34183c5c11 tools/python/xen/xend/XendCheckpoint.py --- a/tools/python/xen/xend/XendCheckpoint.py Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/xend/XendCheckpoint.py Mon Oct 19 15:50:06 2009 -0600 @@ -226,6 +226,19 @@ def restore(xd, fd, dominfo = None, paus else: dominfo = xd.restore_(vmconfig) + image_cfg = dominfo.info.get(''image'', {}) + is_hvm = dominfo.info.is_hvm() + + if is_hvm: + nomigrate = dominfo.info[''platform''].get(''nomigrate'', 0) + else: + nomigrate = dominfo.info[''platform''].get(''nomigrate'') + if nomigrate is None: + nomigrate = 0 + if int(nomigrate) != 0: + dominfo.destroy() + raise XendError("cannot restore non-migratable domain") + # repin domain vcpus if a target node number was specified # this is done prior to memory allocation to aide in memory # distribution for NUMA systems. @@ -248,8 +261,6 @@ def restore(xd, fd, dominfo = None, paus assert console_port # if hvm, pass mem size to calculate the store_mfn - image_cfg = dominfo.info.get(''image'', {}) - is_hvm = dominfo.info.is_hvm() if is_hvm: apic = int(dominfo.info[''platform''].get(''apic'', 0)) pae = int(dominfo.info[''platform''].get(''pae'', 0)) diff -r ea34183c5c11 tools/python/xen/xend/XendConfig.py --- a/tools/python/xen/xend/XendConfig.py Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/xend/XendConfig.py Mon Oct 19 15:50:06 2009 -0600 @@ -146,6 +146,7 @@ XENAPI_PLATFORM_CFG_TYPES = { ''localtime'': int, ''monitor'': int, ''nographic'': int, + ''nomigrate'': int, ''pae'' : int, ''rtc_timeoffset'': int, ''serial'': str, @@ -478,6 +479,9 @@ class XendConfig(dict): if ''tsc_native'' not in self[''platform'']: self[''platform''][''tsc_native''] = 0 + + if ''nomigrate'' not in self[''platform'']: + self[''platform''][''nomigrate''] = 0 if self.is_hvm(): if ''timer_mode'' not in self[''platform'']: diff -r ea34183c5c11 tools/python/xen/xend/XendDomainInfo.py --- a/tools/python/xen/xend/XendDomainInfo.py Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/xend/XendDomainInfo.py Mon Oct 19 15:50:06 2009 -0600 @@ -2421,7 +2421,7 @@ class XendDomainInfo: s3_integrity = self.info[''s3_integrity''] oos = self.info[''platform''].get(''oos'', 1) - oos_off = 1 - oos + oos_off = 1 - int(oos) flags = (int(hvm) << 0) | (int(hap) << 1) | (int(s3_integrity) << 2) | (int(oos_off) << 3) @@ -2463,6 +2463,11 @@ class XendDomainInfo: viridian = self.info["platform"].get("viridian") if arch.type == "x86" and hvm and viridian is not None: xc.hvm_set_param(self.domid, HVM_PARAM_VIRIDIAN, long(viridian)) + + # If nomigrate is set, disable migration + nomigrate = self.info["platform"].get("nomigrate") + if arch.type == "x86" and nomigrate is not None and long(nomigrate) != 0: + xc.domain_disable_migrate(self.domid) # Optionally enable virtual HPET hpet = self.info["platform"].get("hpet") diff -r ea34183c5c11 tools/python/xen/xm/create.py --- a/tools/python/xen/xm/create.py Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/xm/create.py Mon Oct 19 15:50:06 2009 -0600 @@ -224,6 +224,10 @@ gopts.var(''tsc_native'', val=''TSC_NATIVE'' gopts.var(''tsc_native'', val=''TSC_NATIVE'', fn=set_int, default=0, use="""TSC mode (0=emulate TSC, 1=native TSC).""") + +gopts.var(''nomigrate'', val=''NOMIGRATE'', + fn=set_int, default=0, + use="""migratability (0=migration enabled, 1=migration disabled).""") gopts.var(''vpt_align'', val=''VPT_ALIGN'', fn=set_int, default=1, @@ -737,6 +741,9 @@ def configure_image(vals): if vals.tsc_native is not None: config_image.append([''tsc_native'', vals.tsc_native]) + if vals.nomigrate is not None: + config_image.append([''nomigrate'', vals.nomigrate]) + return config_image def configure_disks(config_devs, vals): @@ -1020,7 +1027,7 @@ def make_config(vals): config.append([n, v]) map(add_conf, [''name'', ''memory'', ''maxmem'', ''shadow_memory'', - ''restart'', ''on_poweroff'', ''tsc_native'', + ''restart'', ''on_poweroff'', ''tsc_native'', ''nomigrate'', ''on_reboot'', ''on_crash'', ''vcpus'', ''vcpu_avail'', ''features'', ''on_xend_start'', ''on_xend_stop'', ''target'', ''cpuid'', ''cpuid_check'', ''machine_address_size'', ''suppress_spurious_page_faults'']) diff -r ea34183c5c11 tools/python/xen/xm/xenapi_create.py --- a/tools/python/xen/xm/xenapi_create.py Mon Oct 19 13:31:21 2009 +0100 +++ b/tools/python/xen/xm/xenapi_create.py Mon Oct 19 15:50:06 2009 -0600 @@ -1079,6 +1079,7 @@ class sxp2xml: ''xen_platform_pci'', ''tsc_native'' ''description'', + ''nomigrate'' ] platform_configs = [] diff -r ea34183c5c11 xen/arch/x86/domain.c --- a/xen/arch/x86/domain.c Mon Oct 19 13:31:21 2009 +0100 +++ b/xen/arch/x86/domain.c Mon Oct 19 15:50:06 2009 -0600 @@ -521,6 +521,9 @@ int arch_domain_create(struct domain *d, } spin_lock_init(&d->arch.vtsc_lock); + + if ( d->domain_id == 0 ) + d->disable_migrate = 1; return 0; diff -r ea34183c5c11 xen/arch/x86/domctl.c --- a/xen/arch/x86/domctl.c Mon Oct 19 13:31:21 2009 +0100 +++ b/xen/arch/x86/domctl.c Mon Oct 19 15:50:06 2009 -0600 @@ -1100,6 +1100,24 @@ long arch_do_domctl( } break; + case XEN_DOMCTL_disable_migrate: + { + struct domain *d; + + ret = -ESRCH; + d = rcu_lock_domain_by_id(domctl->domain); + if ( d == NULL ) + break; + + domain_pause(d); + d->arch.disable_migrate = domctl->u.disable_migrate.disable; + domain_unpause(d); + + rcu_unlock_domain(d); + ret = 0; + } + break; + case XEN_DOMCTL_suppress_spurious_page_faults: { struct domain *d; diff -r ea34183c5c11 xen/include/asm-x86/domain.h --- a/xen/include/asm-x86/domain.h Mon Oct 19 13:31:21 2009 +0100 +++ b/xen/include/asm-x86/domain.h Mon Oct 19 15:50:06 2009 -0600 @@ -304,6 +304,9 @@ struct arch_domain spinlock_t vtsc_lock; uint64_t vtsc_kerncount; /* for hvm, counts all vtsc */ uint64_t vtsc_usercount; /* not used for hvm */ + + /* mark domain as non-migratable and non-restoreable */ + bool_t disable_migrate; } __cacheline_aligned; #define has_arch_pdevs(d) (!list_empty(&(d)->arch.pdev_list)) diff -r ea34183c5c11 xen/include/public/domctl.h --- a/xen/include/public/domctl.h Mon Oct 19 13:31:21 2009 +0100 +++ b/xen/include/public/domctl.h Mon Oct 19 15:50:06 2009 -0600 @@ -653,6 +653,12 @@ typedef struct xen_domctl_hvmcontext_par } xen_domctl_hvmcontext_partial_t; DEFINE_XEN_GUEST_HANDLE(xen_domctl_hvmcontext_partial_t); +#define XEN_DOMCTL_disable_migrate 58 +typedef struct xen_domctl_disable_migrate { + uint32_t disable; /* IN: 1: disable migration and restore */ +} xen_domctl_disable_migrate_t; + + #define XEN_DOMCTL_gdbsx_guestmemio 1000 /* guest mem io */ struct xen_domctl_gdbsx_memio { uint64_aligned_t pgd3val;/* optional: init_mm.pgd[3] value */ @@ -703,6 +709,7 @@ struct xen_domctl { struct xen_domctl_arch_setup arch_setup; struct xen_domctl_settimeoffset settimeoffset; struct xen_domctl_set_tsc_native set_tsc_native; + struct xen_domctl_disable_migrate disable_migrate; struct xen_domctl_real_mode_area real_mode_area; struct xen_domctl_hvmcontext hvmcontext; struct xen_domctl_hvmcontext_partial hvmcontext_partial; _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
Hi, At 23:11 +0100 on 19 Oct (1255993888), Dan Magenheimer wrote:> The option persists into a save file > and is also communicated into the hypervisor, the latter > for the purposes of a to-be-added hypercall for communicating > to guests that migration is disallowedCan you please use Xenstore for that? There''s no reason for the hypervisor to be involved, since all the enforcememnt is in the tools. Cheers, Tim> (which will be > used initially for userland TSC-related sensing, but may > find other uses). > > Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com> >-- Tim Deegan <Tim.Deegan@citrix.com> Principal Software Engineer, Citrix Systems (R&D) Ltd. [Company #02300071, SL9 0DZ, UK.] _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
On 20/10/2009 08:43, "Tim Deegan" <Tim.Deegan@citrix.com> wrote:> At 23:11 +0100 on 19 Oct (1255993888), Dan Magenheimer wrote: >> The option persists into a save file >> and is also communicated into the hypervisor, the latter >> for the purposes of a to-be-added hypercall for communicating >> to guests that migration is disallowed > > Can you please use Xenstore for that? There''s no reason for the > hypervisor to be involved, since all the enforcememnt is in the tools.I suspect Dan wants to interrogate this flag from unprivileged domU userspace, with no domU kernel modifications. Make of that what you will. ;-) -- Keir _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
xen@flonatel.org
2009-Oct-20 09:01 UTC
Re: [Xen-devel] [PATCH] new nomigrate config option
Hello! I like this feature - but it (CS 20344: d61a0c986a68) does not compile for me. Is is possible that there is a lost indirection in the domain.c? (I''m very unsure about this patch: it''s only compile-tested.) Kind regards Andreas diff -r d61a0c986a68 xen/arch/x86/domain.c --- a/xen/arch/x86/domain.c Tue Oct 20 08:45:12 2009 +0100 +++ b/xen/arch/x86/domain.c Tue Oct 20 10:59:35 2009 +0200 @@ -523,7 +523,7 @@ spin_lock_init(&d->arch.vtsc_lock); if ( d->domain_id == 0 ) - d->disable_migrate = 1; + d->arch.disable_migrate = 1; return 0; _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
At 08:54 +0100 on 20 Oct (1256028889), Keir Fraser wrote:> On 20/10/2009 08:43, "Tim Deegan" <Tim.Deegan@citrix.com> wrote: > > > At 23:11 +0100 on 19 Oct (1255993888), Dan Magenheimer wrote: > >> The option persists into a save file > >> and is also communicated into the hypervisor, the latter > >> for the purposes of a to-be-added hypercall for communicating > >> to guests that migration is disallowed > > > > Can you please use Xenstore for that? There''s no reason for the > > hypervisor to be involved, since all the enforcememnt is in the tools. > > I suspect Dan wants to interrogate this flag from unprivileged domU > userspace, with no domU kernel modifications.In that case all that''s needed is a user-space xenbus client. :) Tim. -- Tim Deegan <Tim.Deegan@citrix.com> Principal Software Engineer, Citrix Systems (R&D) Ltd. [Company #02300071, SL9 0DZ, UK.] _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
On 20/10/2009 10:01, "xen@flonatel.org" <xen@flonatel.org> wrote:> Hello! > > I like this feature - but it (CS 20344: d61a0c986a68) does not compile for me. > > Is is possible that there is a lost indirection in the domain.c? (I''m > very unsure about this patch: it''s only compile-tested.)Thanks. Now fixed. -- Keir _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
On 20/10/2009 10:14, "Tim Deegan" <Tim.Deegan@eu.citrix.com> wrote:>>> Can you please use Xenstore for that? There''s no reason for the >>> hypervisor to be involved, since all the enforcememnt is in the tools. >> >> I suspect Dan wants to interrogate this flag from unprivileged domU >> userspace, with no domU kernel modifications. > > In that case all that''s needed is a user-space xenbus client. :)Is that all. :-) I plan to see how this userspace hypercall stuff goes and at the end of the day, if/when we don''t like it, it''s going to be very easy to disable. -- Keir _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
> >> I suspect Dan wants to interrogate this flag from unprivileged domU > >> userspace, with no domU kernel modifications. > > > > In that case all that''s needed is a user-space xenbus client. :) > > Is that all. :-) > > I plan to see how this userspace hypercall stuff goes and at > the end of the > day, if/when we don''t like it, it''s going to be very easy to disable.Actually, I''ve got an interesting idea that requires no userspace hypercall. If the physical system has the Invariant TSC cpuid bit set AND a domain has nomigrate==1, the Invariant TSC bit can/should be surfaced to the domain. Also, Invariant TSC can/should be surfaced to ANY domain that is booted with tsc_native==0. Then apps that test for the Invariant TSC bit (and find it set) can always use rdtsc directly. More complicated environments (e.g. where high-frequency time stamp apps are running and migration is still required) may still need userspace hypercall, but this first step can still be useful on many physical systems without them. Dan _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel
Oops, sorry I forgot to add perhaps the most important part of the idea below: If the physical system has the Invariant TSC cpuid bit set AND a domain has nomigrate==1, tsc_native could be auto-enabled for that domain!> -----Original Message----- > From: Dan Magenheimer > Sent: Thursday, October 22, 2009 11:57 AM > To: Keir Fraser; Tim Deegan > Cc: Xen-Devel (E-mail) > Subject: RE: [Xen-devel] [PATCH] new nomigrate config option > > > > >> I suspect Dan wants to interrogate this flag from > unprivileged domU > > >> userspace, with no domU kernel modifications. > > > > > > In that case all that''s needed is a user-space xenbus client. :) > > > > Is that all. :-) > > > > I plan to see how this userspace hypercall stuff goes and at > > the end of the > > day, if/when we don''t like it, it''s going to be very easy > to disable. > > Actually, I''ve got an interesting idea that requires no > userspace hypercall. > > If the physical system has the Invariant TSC cpuid bit set > AND a domain has nomigrate==1, the Invariant TSC bit can/should > be surfaced to the domain. Also, Invariant TSC can/should > be surfaced to ANY domain that is booted with tsc_native==0. > > Then apps that test for the Invariant TSC bit (and find it > set) can always use rdtsc directly. > > More complicated environments (e.g. where high-frequency time > stamp apps are running and migration is still required) may > still need userspace hypercall, but this first step can > still be useful on many physical systems without them. > > Dan > > _______________________________________________ > Xen-devel mailing list > Xen-devel@lists.xensource.com > http://lists.xensource.com/xen-devel >_______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel