Xen devel - May 2013 - [PATCH 2/2] x86/xen: sync the wallclock when the system time changes

From: David Vrabel <david.vrabel@citrix.com>

Currently the Xen wallclock is only updated every 11 minutes if NTP is
synchronized to its clock source.  If a guest is started before NTP is
synchronized it may see an incorrect wallclock time.

Use the pvclock_gtod notifier chain to receive a notification when the
system time has changed and update the wallclock to match.

This chain is called on every timer tick and we want to avoid an extra
(expensive) hypercall on every tick.  Because dom0 has historically
never provided a very accurate wallclock and guests do not expect one,
we can do this simply.  The wallclock is only updated if the
difference between now and the last update is more than 0.5 s.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 arch/x86/xen/time.c |   54 ++++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index 4656165..81027b5 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -15,6 +15,8 @@
 #include <linux/math64.h>
 #include <linux/gfp.h>
 #include <linux/mc146818rtc.h>
+#include <linux/timekeeper_internal.h>
+#include <linux/pvclock_gtod.h>
 
 #include <asm/pvclock.h>
 #include <asm/xen/hypervisor.h>
@@ -199,28 +201,59 @@ static void xen_get_wallclock(struct timespec *now)
 
 static int xen_set_wallclock(const struct timespec *now)
 {
-	struct xen_platform_op op;
-	int ret;
-
 	/* do nothing for domU */
 	if (!xen_initial_domain())
 		return -1;
 
-	/* Set the Xen wallclock. */
+	/* Set the hardware RTC. */
+	return mach_set_rtc_mmss(now);
+
+}
+
+static int xen_pvclock_gtod_notify(struct notifier_block *nb, unsigned long
unused,
+				   void *priv)
+{
+	static struct timespec last, next;
+	struct timespec now;
+	struct timekeeper *tk = priv;
+	struct xen_platform_op op;
+	int ret;
+
+	/*
+	 * Set the Xen wallclock from Linux system time.
+	 *
+	 * dom0 hasn''t historically maintained a very accurate
+	 * wallclock so guests don''t expect it. We can therefore
+	 * reduce the number of expensive hypercalls by only updating
+	 * the wallclock every 0.5 s.
+	 */
+
+	now.tv_sec = tk->xtime_sec;
+	now.tv_nsec = tk->xtime_nsec >> tk->shift;
+
+	if (timespec_compare(&now, &last) > 0
+	    && timespec_compare(&now, &next) < 0)
+		return 0;
+
 	op.cmd = XENPF_settime;
-	op.u.settime.secs = now->tv_sec;
-	op.u.settime.nsecs = now->tv_nsec;
+	op.u.settime.secs = now.tv_sec;
+	op.u.settime.nsecs = now.tv_nsec;
 	op.u.settime.system_time = xen_clocksource_read();
 
 	ret = HYPERVISOR_dom0_op(&op);
 	if (ret)
-		return ret;
+		return 0;
 
-	/* Set the hardware RTC. */
-	return mach_set_rtc_mmss(now);
+	last = now;
+	next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
 
+	return 0;
 }
 
+static struct notifier_block xen_pvclock_gtod_notifier = {
+	.notifier_call = xen_pvclock_gtod_notify,
+};
+
 static struct clocksource xen_clocksource __read_mostly = {
 	.name = "xen",
 	.rating = 400,
@@ -482,6 +515,9 @@ static void __init xen_time_init(void)
 	xen_setup_runstate_info(cpu);
 	xen_setup_timer(cpu);
 	xen_setup_cpu_clockevents();
+
+	if (xen_initial_domain())
+		pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
 }
 
 void __init xen_init_time_ops(void)
-- 
1.7.2.5

On 05/28/2013 11:22 AM, David Vrabel wrote:
> From: David Vrabel <david.vrabel@citrix.com>
>
> Currently the Xen wallclock is only updated every 11 minutes if NTP is
> synchronized to its clock source.  If a guest is started before NTP is
> synchronized it may see an incorrect wallclock time.
>
> Use the pvclock_gtod notifier chain to receive a notification when the
> system time has changed and update the wallclock to match.
>
> This chain is called on every timer tick and we want to avoid an extra
> (expensive) hypercall on every tick.  Because dom0 has historically
> never provided a very accurate wallclock and guests do not expect one,
> we can do this simply.  The wallclock is only updated if the
> difference between now and the last update is more than 0.5 s.
>
> Signed-off-by: David Vrabel <david.vrabel@citrix.com>
> ---
>   arch/x86/xen/time.c |   54
++++++++++++++++++++++++++++++++++++++++++--------
>   1 files changed, 45 insertions(+), 9 deletions(-)
>
> diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
> index 4656165..81027b5 100644
> --- a/arch/x86/xen/time.c
> +++ b/arch/x86/xen/time.c
> @@ -15,6 +15,8 @@
>   #include <linux/math64.h>
>   #include <linux/gfp.h>
>   #include <linux/mc146818rtc.h>
> +#include <linux/timekeeper_internal.h>
Nope. You shouldn''t be including this.
> +#include <linux/pvclock_gtod.h>
>   
>   #include <asm/pvclock.h>
>   #include <asm/xen/hypervisor.h>
> @@ -199,28 +201,59 @@ static void xen_get_wallclock(struct timespec *now)
>   
>   static int xen_set_wallclock(const struct timespec *now)
>   {
> -	struct xen_platform_op op;
> -	int ret;
> -
>   	/* do nothing for domU */
>   	if (!xen_initial_domain())
>   		return -1;
>   
> -	/* Set the Xen wallclock. */
> +	/* Set the hardware RTC. */
> +	return mach_set_rtc_mmss(now);
> +
> +}
> +
> +static int xen_pvclock_gtod_notify(struct notifier_block *nb, unsigned
long unused,
> +				   void *priv)
> +{
> +	static struct timespec last, next;
> +	struct timespec now;
> +	struct timekeeper *tk = priv;
Noooo.. don''t export the timekeeper like that.

> +	struct xen_platform_op op;
> +	int ret;
> +
> +	/*
> +	 * Set the Xen wallclock from Linux system time.
> +	 *
> +	 * dom0 hasn''t historically maintained a very accurate
> +	 * wallclock so guests don''t expect it. We can therefore
> +	 * reduce the number of expensive hypercalls by only updating
> +	 * the wallclock every 0.5 s.
> +	 */
> +
> +	now.tv_sec = tk->xtime_sec;
> +	now.tv_nsec = tk->xtime_nsec >> tk->shift;
You probably want current_kernel_time() here.
> +
> +	if (timespec_compare(&now, &last) > 0
> +	    && timespec_compare(&now, &next) < 0)
> +		return 0;
> +
>   	op.cmd = XENPF_settime;
> -	op.u.settime.secs = now->tv_sec;
> -	op.u.settime.nsecs = now->tv_nsec;
> +	op.u.settime.secs = now.tv_sec;
> +	op.u.settime.nsecs = now.tv_nsec;
>   	op.u.settime.system_time = xen_clocksource_read();
>   
>   	ret = HYPERVISOR_dom0_op(&op);
>   	if (ret)
> -		return ret;
> +		return 0;
>   
> -	/* Set the hardware RTC. */
> -	return mach_set_rtc_mmss(now);
> +	last = now;
> +	next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
>   
Am I missing the xen_set_wallclock hook here? Your previous patch wanted 
to call the dom0 op and then set the hardware RTC.

And this notifier get called every timer tick? This seems out there...

thanks
-john

>>> On 28.05.13 at 20:22, David Vrabel <david.vrabel@citrix.com>
wrote:
> +static int xen_pvclock_gtod_notify(struct notifier_block *nb, unsigned
long unused,
> +				   void *priv)
> +{
> +	static struct timespec last, next;
> +	struct timespec now;
> +	struct timekeeper *tk = priv;
> +	struct xen_platform_op op;
> +	int ret;
> +
> +	/*
> +	 * Set the Xen wallclock from Linux system time.
> +	 *
> +	 * dom0 hasn''t historically maintained a very accurate
> +	 * wallclock so guests don''t expect it. We can therefore
> +	 * reduce the number of expensive hypercalls by only updating
> +	 * the wallclock every 0.5 s.
> +	 */
> +
> +	now.tv_sec = tk->xtime_sec;
> +	now.tv_nsec = tk->xtime_nsec >> tk->shift;
> +
> +	if (timespec_compare(&now, &last) > 0
> +	    && timespec_compare(&now, &next) < 0)
Is this really working the first time through (when both last and
next are still all zeros)?

Jan
> +		return 0;
> +
>  	op.cmd = XENPF_settime;
> -	op.u.settime.secs = now->tv_sec;
> -	op.u.settime.nsecs = now->tv_nsec;
> +	op.u.settime.secs = now.tv_sec;
> +	op.u.settime.nsecs = now.tv_nsec;
>  	op.u.settime.system_time = xen_clocksource_read();
>  
>  	ret = HYPERVISOR_dom0_op(&op);
>  	if (ret)
> -		return ret;
> +		return 0;
>  
> -	/* Set the hardware RTC. */
> -	return mach_set_rtc_mmss(now);
> +	last = now;
> +	next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
>  
> +	return 0;
>  }

>>> On 28.05.13 at 20:53, John Stultz <john.stultz@linaro.org>
wrote:
> On 05/28/2013 11:22 AM, David Vrabel wrote:
>>   	op.cmd = XENPF_settime;
>> -	op.u.settime.secs = now->tv_sec;
>> -	op.u.settime.nsecs = now->tv_nsec;
>> +	op.u.settime.secs = now.tv_sec;
>> +	op.u.settime.nsecs = now.tv_nsec;
>>   	op.u.settime.system_time = xen_clocksource_read();
>>   
>>   	ret = HYPERVISOR_dom0_op(&op);
>>   	if (ret)
>> -		return ret;
>> +		return 0;
>>   
>> -	/* Set the hardware RTC. */
>> -	return mach_set_rtc_mmss(now);
>> +	last = now;
>> +	next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
>>   
> 
> Am I missing the xen_set_wallclock hook here? Your previous patch wanted 
> to call the dom0 op and then set the hardware RTC.
That ought to be intentional: We should update the hypervisor''s
software wall clock here, but limit updating of the RTC to whenever
a native kernel would do this (i.e. among other things subject to
CONFIG_RTC_SYSTOHC).

Jan

>>> On 28.05.13 at 20:22, David Vrabel <david.vrabel@citrix.com>
wrote:
> @@ -199,28 +201,59 @@ static void xen_get_wallclock(struct timespec *now)
>  
>  static int xen_set_wallclock(const struct timespec *now)
>  {
> -	struct xen_platform_op op;
> -	int ret;
> -
>  	/* do nothing for domU */
>  	if (!xen_initial_domain())
>  		return -1;
>  
> -	/* Set the Xen wallclock. */
> +	/* Set the hardware RTC. */
> +	return mach_set_rtc_mmss(now);
> +
> +}
Can''t you achieve the same effect in a cleaner way by overriding
x86_platform.set_wallclock only in the non-init-domain case?

Jan

On 29/05/13 08:47, Jan Beulich wrote:
>>>> On 28.05.13 at 20:22, David Vrabel
<david.vrabel@citrix.com> wrote:
>> @@ -199,28 +201,59 @@ static void xen_get_wallclock(struct timespec
*now)
>>  
>>  static int xen_set_wallclock(const struct timespec *now)
>>  {
>> -	struct xen_platform_op op;
>> -	int ret;
>> -
>>  	/* do nothing for domU */
>>  	if (!xen_initial_domain())
>>  		return -1;
>>  
>> -	/* Set the Xen wallclock. */
>> +	/* Set the hardware RTC. */
>> +	return mach_set_rtc_mmss(now);
>> +
>> +}
> 
> Can''t you achieve the same effect in a cleaner way by overriding
> x86_platform.set_wallclock only in the non-init-domain case?
Yes, that makes sense.  John, don''t apply this one yet, thanks.

David

On 28/05/13 19:53, John Stultz wrote:
> On 05/28/2013 11:22 AM, David Vrabel wrote:
>> From: David Vrabel <david.vrabel@citrix.com>
>>
>> Currently the Xen wallclock is only updated every 11 minutes if NTP is
>> synchronized to its clock source.  If a guest is started before NTP is
>> synchronized it may see an incorrect wallclock time.
>>
>> Use the pvclock_gtod notifier chain to receive a notification when the
>> system time has changed and update the wallclock to match.
>>
>> This chain is called on every timer tick and we want to avoid an extra
>> (expensive) hypercall on every tick.  Because dom0 has historically
>> never provided a very accurate wallclock and guests do not expect one,
>> we can do this simply.  The wallclock is only updated if the
>> difference between now and the last update is more than 0.5 s.
[...]
>> index 4656165..81027b5 100644
>> --- a/arch/x86/xen/time.c
>> +++ b/arch/x86/xen/time.c
[...]
>> +    struct xen_platform_op op;
>> +    int ret;
>> +
>> +    /*
>> +     * Set the Xen wallclock from Linux system time.
>> +     *
>> +     * dom0 hasn''t historically maintained a very accurate
>> +     * wallclock so guests don''t expect it. We can therefore
>> +     * reduce the number of expensive hypercalls by only updating
>> +     * the wallclock every 0.5 s.
>> +     */
>> +
>> +    now.tv_sec = tk->xtime_sec;
>> +    now.tv_nsec = tk->xtime_nsec >> tk->shift;
> 
> You probably want current_kernel_time() here.
Ah. I was looking for something like this but since the notifier chain
is called with various locks held stuff kept deadlocking.  It looks like
I can use __current_kernel_time() though.
>> +
>> +    if (timespec_compare(&now, &last) > 0
>> +        && timespec_compare(&now, &next) < 0)
>> +        return 0;
>> +
>>       op.cmd = XENPF_settime;
>> -    op.u.settime.secs = now->tv_sec;
>> -    op.u.settime.nsecs = now->tv_nsec;
>> +    op.u.settime.secs = now.tv_sec;
>> +    op.u.settime.nsecs = now.tv_nsec;
>>       op.u.settime.system_time = xen_clocksource_read();
>>         ret = HYPERVISOR_dom0_op(&op);
>>       if (ret)
>> -        return ret;
>> +        return 0;
>>   -    /* Set the hardware RTC. */
>> -    return mach_set_rtc_mmss(now);
>> +    last = now;
>> +    next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
>>   
> 
> Am I missing the xen_set_wallclock hook here? Your previous patch wanted
> to call the dom0 op and then set the hardware RTC.
As Jan says, this is deliberate.  We now update the hardware CMOS RTC in
the same way as bare metal and then use this notifier to only maintain
the Xen wallclock for guests.
> And this notifier get called every timer tick? This seems out there...
This notifier was added to support similar functionality for KVM and
I''m
just reusing what''s there.

See e0b306fef (time: export time information for KVM pvclock) [1] and
16e8d74d2 (KVM: x86: notifier for clocksource changes) [2].

Perhaps Marcelo can comment on why it was necessary for this to be
called on every timer tick.

David

[1]
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=e0b306fef90556233797d2e1747bd6a3ae35ea93
[2]
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=16e8d74d2da9920f874b10a3d979fb25c01f518f

On 29/05/13 08:39, Jan Beulich wrote:
>>>> On 28.05.13 at 20:22, David Vrabel
<david.vrabel@citrix.com> wrote:
>> +static int xen_pvclock_gtod_notify(struct notifier_block *nb, unsigned
long unused,
>> +				   void *priv)
>> +{
>> +	static struct timespec last, next;
>> +	struct timespec now;
>> +	struct timekeeper *tk = priv;
>> +	struct xen_platform_op op;
>> +	int ret;
>> +
>> +	/*
>> +	 * Set the Xen wallclock from Linux system time.
>> +	 *
>> +	 * dom0 hasn''t historically maintained a very accurate
>> +	 * wallclock so guests don''t expect it. We can therefore
>> +	 * reduce the number of expensive hypercalls by only updating
>> +	 * the wallclock every 0.5 s.
>> +	 */
>> +
>> +	now.tv_sec = tk->xtime_sec;
>> +	now.tv_nsec = tk->xtime_nsec >> tk->shift;
>> +
>> +	if (timespec_compare(&now, &last) > 0
>> +	    && timespec_compare(&now, &next) < 0)
> 
> Is this really working the first time through (when both last and
> next are still all zeros)?
Yes. The first time through we want to set the wallclock and this test
is always false when last == next.

David

On 05/29/2013 02:37 AM, David Vrabel wrote:
> On 29/05/13 08:47, Jan Beulich wrote:
>>>>> On 28.05.13 at 20:22, David Vrabel
<david.vrabel@citrix.com> wrote:
>>> @@ -199,28 +201,59 @@ static void xen_get_wallclock(struct timespec
*now)
>>>   
>>>   static int xen_set_wallclock(const struct timespec *now)
>>>   {
>>> -	struct xen_platform_op op;
>>> -	int ret;
>>> -
>>>   	/* do nothing for domU */
>>>   	if (!xen_initial_domain())
>>>   		return -1;
>>>   
>>> -	/* Set the Xen wallclock. */
>>> +	/* Set the hardware RTC. */
>>> +	return mach_set_rtc_mmss(now);
>>> +
>>> +}
>> Can''t you achieve the same effect in a cleaner way by
overriding
>> x86_platform.set_wallclock only in the non-init-domain case?
> Yes, that makes sense.  John, don''t apply this one yet, thanks.
Ok. Zapped from my tree.

thanks
-john

On Wed, May 29, 2013 at 10:42:04AM +0100, David Vrabel
wrote:
> On 28/05/13 19:53, John Stultz wrote:
> > On 05/28/2013 11:22 AM, David Vrabel wrote:
> >> From: David Vrabel <david.vrabel@citrix.com>
> >>
> >> Currently the Xen wallclock is only updated every 11 minutes if
NTP is
> >> synchronized to its clock source.  If a guest is started before
NTP is
> >> synchronized it may see an incorrect wallclock time.
> >>
> >> Use the pvclock_gtod notifier chain to receive a notification when
the
> >> system time has changed and update the wallclock to match.
> >>
> >> This chain is called on every timer tick and we want to avoid an
extra
> >> (expensive) hypercall on every tick.  Because dom0 has
historically
> >> never provided a very accurate wallclock and guests do not expect
one,
> >> we can do this simply.  The wallclock is only updated if the
> >> difference between now and the last update is more than 0.5 s.
> [...]
> >> index 4656165..81027b5 100644
> >> --- a/arch/x86/xen/time.c
> >> +++ b/arch/x86/xen/time.c
> [...]
> >> +    struct xen_platform_op op;
> >> +    int ret;
> >> +
> >> +    /*
> >> +     * Set the Xen wallclock from Linux system time.
> >> +     *
> >> +     * dom0 hasn''t historically maintained a very
accurate
> >> +     * wallclock so guests don''t expect it. We can
therefore
> >> +     * reduce the number of expensive hypercalls by only updating
> >> +     * the wallclock every 0.5 s.
> >> +     */
> >> +
> >> +    now.tv_sec = tk->xtime_sec;
> >> +    now.tv_nsec = tk->xtime_nsec >> tk->shift;
> > 
> > You probably want current_kernel_time() here.
> 
> Ah. I was looking for something like this but since the notifier chain
> is called with various locks held stuff kept deadlocking.  It looks like
> I can use __current_kernel_time() though.
> 
> >> +
> >>  +    if (timespec_compare(&now, &last) > 0
> >> +        && timespec_compare(&now, &next) < 0)
> >> +        return 0;
> >> +
> >>       op.cmd = XENPF_settime;
> >> -    op.u.settime.secs = now->tv_sec;
> >> -    op.u.settime.nsecs = now->tv_nsec;
> >> +    op.u.settime.secs = now.tv_sec;
> >> +    op.u.settime.nsecs = now.tv_nsec;
> >>       op.u.settime.system_time = xen_clocksource_read();
> >>         ret = HYPERVISOR_dom0_op(&op);
> >>       if (ret)
> >> -        return ret;
> >> +        return 0;
> >>   -    /* Set the hardware RTC. */
> >> -    return mach_set_rtc_mmss(now);
> >> +    last = now;
> >> +    next = timespec_add(now, ns_to_timespec(NSEC_PER_SEC / 2));
> >>   
> > 
> > Am I missing the xen_set_wallclock hook here? Your previous patch
wanted
> > to call the dom0 op and then set the hardware RTC.
> 
> As Jan says, this is deliberate.  We now update the hardware CMOS RTC in
> the same way as bare metal and then use this notifier to only maintain
> the Xen wallclock for guests.
> 
> > And this notifier get called every timer tick? This seems out there...
> 
> This notifier was added to support similar functionality for KVM and
I''m
> just reusing what''s there.
> 
> See e0b306fef (time: export time information for KVM pvclock) [1] and
> 16e8d74d2 (KVM: x86: notifier for clocksource changes) [2].
> 
> Perhaps Marcelo can comment on why it was necessary for this to be
> called on every timer tick.
Just so to maintain similarity with the vsyscall notifier (as in updates
all fields the vsyscall notifier uses).

KVM only actually makes use of the clocksource change case.