Tim Deegan
2012-Mar-15 14:15 UTC
[PATCH] arm: don''t use atomic operations to gate non-boot CPUs
# HG changeset patch
# User Tim Deegan <tim@xen.org>
# Date 1331820837 0
# Node ID 6f52e5de1b942432ffa6c3b8d194e5e2771c1957
# Parent 4da1453ed61c28a366162b49b2f59f62e070a799
arm: don''t use atomic operations to gate non-boot CPUs.
Since the cache is not enabled that early, better not to rely on
load-linked/store-conditional. Instead, have the boot CPU call the
other CPUs by their IDs, just like we do later for proper CPU bringup.
Signed-off-by: Tim Deegan <tim@xen.org>
diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/head.S
--- a/xen/arch/arm/head.S Thu Mar 15 11:47:27 2012 +0000
+++ b/xen/arch/arm/head.S Thu Mar 15 14:13:57 2012 +0000
@@ -71,16 +71,13 @@ start:
bics r12, r0, #(0xff << 24) /* Mask out flags to get CPU ID */
beq boot_cpu /* If we''re CPU 0, boot now */
- /* Non-boot CPUs wait here to be woken up one at a time.
- * This is basically an open-coded spin-lock to serialize. */
- ldr r0, =boot_gate /* VA of gate */
+ /* Non-boot CPUs wait here to be woken up one at a time. */
+1: wfe
+ dsb
+ ldr r0, =smp_up_cpu /* VA of gate */
add r0, r0, r10 /* PA of gate */
- mov r1, #1 /* (1 == locked) */
-1: wfe
- ldrex r2, [r0] /* Linked read of current value */
- teq r2, #0 /* (0 == unlocked) */
- strexeq r2, r1, [r0] /* Matching update -> locked */
- teq r2, #0 /* (0 == succeeded) */
+ ldr r1, [r0] /* Which CPU is being booted? */
+ teq r1, r12 /* Is it us? */
bne 1b
boot_cpu:
@@ -270,16 +267,7 @@ paging:
teq r12, #0
beq launch
- /* Signal the next non-boot CPU to come and join us here */
- ldr r0, =boot_gate /* VA of gate */
- add r0, r0, r10 /* PA of gate */
- mov r1, #0 /* (0 == unlocked) */
- str r1, [r0]
- dsb
- isb
- sev
-
- /* Move on to the relocated pagetables */
+ /* Non-boot CPUs need to move on to the relocated pagetables */
mov r0, #0
ldr r4, =boot_httbr /* VA of HTTBR value stashed by CPU 0 */
add r4, r4, r10 /* PA of it */
diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/setup.c
--- a/xen/arch/arm/setup.c Thu Mar 15 11:47:27 2012 +0000
+++ b/xen/arch/arm/setup.c Thu Mar 15 14:13:57 2012 +0000
@@ -38,11 +38,6 @@
#include <asm/setup.h>
#include "gic.h"
-/* Spinlock for serializing CPU bringup */
-unsigned long __initdata boot_gate = 1;
-/* Number of non-boot CPUs ready to enter C */
-unsigned long __initdata ready_cpus = 0;
-
static __attribute_used__ void init_done(void)
{
free_init_memory();
@@ -152,8 +147,6 @@ void __init start_xen(unsigned long boot
void *fdt;
size_t fdt_size;
int cpus, i;
- paddr_t gate_pa;
- unsigned long *gate;
fdt = (void *)BOOT_MISC_VIRT_START
+ (atag_paddr & ((1 << SECOND_SHIFT) - 1));
@@ -169,25 +162,11 @@ void __init start_xen(unsigned long boot
console_init_preirq();
#endif
+ cpus = gic_init();
+ make_cpus_ready(cpus, boot_phys_offset);
+
percpu_init_areas();
set_processor_id(0); /* needed early, for smp_processor_id() */
-
- cpus = gic_init();
-
- printk("Waiting for %i other CPUs to be ready\n", cpus - 1);
- /* Bring the other CPUs up to paging before the original
- * copy of .text gets overwritten. We need to use the unrelocated
- * copy of boot_gate as that''s the one the others can see. */
- gate_pa = ((unsigned long) &boot_gate) + boot_phys_offset;
- gate = map_domain_page(gate_pa >> PAGE_SHIFT) + (gate_pa &
~PAGE_MASK);
- *gate = 0;
- unmap_domain_page(gate);
- /* Now send an event to wake the first non-boot CPU */
- asm volatile("dsb; isb; sev");
- /* And wait for them all to be ready. */
- while ( ready_cpus + 1 < cpus )
- smp_rmb();
-
__set_current((struct vcpu *)0xfffff000); /* debug sanity */
idle_vcpu[0] = current;
diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/smpboot.c
--- a/xen/arch/arm/smpboot.c Thu Mar 15 11:47:27 2012 +0000
+++ b/xen/arch/arm/smpboot.c Thu Mar 15 14:13:57 2012 +0000
@@ -19,6 +19,7 @@
#include <xen/cpu.h>
#include <xen/cpumask.h>
#include <xen/delay.h>
+#include <xen/domain_page.h>
#include <xen/errno.h>
#include <xen/init.h>
#include <xen/mm.h>
@@ -41,11 +42,17 @@ static unsigned char __initdata cpu0_boo
/* Pointer to the stack, used by head.S when entering C */
unsigned char *init_stack = cpu0_boot_stack;
+/* Shared state for coordinating CPU bringup */
+unsigned long smp_up_cpu = 0;
+static bool_t cpu_is_dead = 0;
+
+/* Number of non-boot CPUs ready to enter C */
+unsigned long __initdata ready_cpus = 0;
+
void __init
smp_prepare_cpus (unsigned int max_cpus)
{
int i;
- set_processor_id(0); /* needed early, for smp_processor_id() */
cpumask_clear(&cpu_online_map);
cpumask_set_cpu(0, &cpu_online_map);
@@ -56,9 +63,30 @@ smp_prepare_cpus (unsigned int max_cpus)
cpumask_copy(&cpu_present_map, &cpu_possible_map);
}
-/* Shared state for coordinating CPU bringup */
-unsigned long smp_up_cpu = 0;
-static bool_t cpu_is_dead = 0;
+void __init
+make_cpus_ready(unsigned int max_cpus, unsigned long boot_phys_offset)
+{
+ unsigned long *gate;
+ paddr_t gate_pa;
+ int i;
+
+ printk("Waiting for %i other CPUs to be ready\n", max_cpus - 1);
+ /* We use the unrelocated copy of smp_up_cpu as that''s the one the
+ * others can see. */
+ gate_pa = ((paddr_t) (unsigned long) &smp_up_cpu) + boot_phys_offset;
+ gate = map_domain_page(gate_pa >> PAGE_SHIFT) + (gate_pa &
~PAGE_MASK);
+ for ( i = 1; i < max_cpus; i++ )
+ {
+ /* Tell the next CPU to get ready */
+ /* TODO: handle boards where CPUIDs are not contiguous */
+ *gate = i;
+ asm volatile("dsb; isb; sev");
+ /* And wait for it to respond */
+ while ( ready_cpus < i )
+ smp_rmb();
+ }
+ unmap_domain_page(gate);
+}
/* Boot the current CPU */
void __cpuinit start_secondary(unsigned long boot_phys_offset,
diff -r 4da1453ed61c -r 6f52e5de1b94 xen/include/asm-arm/smp.h
--- a/xen/include/asm-arm/smp.h Thu Mar 15 11:47:27 2012 +0000
+++ b/xen/include/asm-arm/smp.h Thu Mar 15 14:13:57 2012 +0000
@@ -16,6 +16,12 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_core_
extern void stop_cpu(void);
+/* Bring the non-boot CPUs up to paging and ready to enter C.
+ * Must be called after Xen is relocated but before the original copy of
+ * .text gets overwritten. */
+extern void
+make_cpus_ready(unsigned int max_cpus, unsigned long boot_phys_offset);
+
#endif
/*
* Local variables:
Ian Campbell
2012-Mar-15 15:21 UTC
Re: [PATCH] arm: don''t use atomic operations to gate non-boot CPUs
On Thu, 2012-03-15 at 14:15 +0000, Tim Deegan wrote:> # HG changeset patch > # User Tim Deegan <tim@xen.org> > # Date 1331820837 0 > # Node ID 6f52e5de1b942432ffa6c3b8d194e5e2771c1957 > # Parent 4da1453ed61c28a366162b49b2f59f62e070a799 > arm: don''t use atomic operations to gate non-boot CPUs. > > Since the cache is not enabled that early, better not to rely on > load-linked/store-conditional. Instead, have the boot CPU call the > other CPUs by their IDs, just like we do later for proper CPU bringup. > > Signed-off-by: Tim Deegan <tim@xen.org>Committed, thanks. (do we care about trailing whitespace? there was some, "git am" warns me...)> > diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/head.S > --- a/xen/arch/arm/head.S Thu Mar 15 11:47:27 2012 +0000 > +++ b/xen/arch/arm/head.S Thu Mar 15 14:13:57 2012 +0000 > @@ -71,16 +71,13 @@ start: > bics r12, r0, #(0xff << 24) /* Mask out flags to get CPU ID */ > beq boot_cpu /* If we''re CPU 0, boot now */ > > - /* Non-boot CPUs wait here to be woken up one at a time. > - * This is basically an open-coded spin-lock to serialize. */ > - ldr r0, =boot_gate /* VA of gate */ > + /* Non-boot CPUs wait here to be woken up one at a time. */ > +1: wfe > + dsb > + ldr r0, =smp_up_cpu /* VA of gate */ > add r0, r0, r10 /* PA of gate */ > - mov r1, #1 /* (1 == locked) */ > -1: wfe > - ldrex r2, [r0] /* Linked read of current value */ > - teq r2, #0 /* (0 == unlocked) */ > - strexeq r2, r1, [r0] /* Matching update -> locked */ > - teq r2, #0 /* (0 == succeeded) */ > + ldr r1, [r0] /* Which CPU is being booted? */ > + teq r1, r12 /* Is it us? */ > bne 1b > > boot_cpu: > @@ -270,16 +267,7 @@ paging: > teq r12, #0 > beq launch > > - /* Signal the next non-boot CPU to come and join us here */ > - ldr r0, =boot_gate /* VA of gate */ > - add r0, r0, r10 /* PA of gate */ > - mov r1, #0 /* (0 == unlocked) */ > - str r1, [r0] > - dsb > - isb > - sev > - > - /* Move on to the relocated pagetables */ > + /* Non-boot CPUs need to move on to the relocated pagetables */ > mov r0, #0 > ldr r4, =boot_httbr /* VA of HTTBR value stashed by CPU 0 */ > add r4, r4, r10 /* PA of it */ > diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/setup.c > --- a/xen/arch/arm/setup.c Thu Mar 15 11:47:27 2012 +0000 > +++ b/xen/arch/arm/setup.c Thu Mar 15 14:13:57 2012 +0000 > @@ -38,11 +38,6 @@ > #include <asm/setup.h> > #include "gic.h" > > -/* Spinlock for serializing CPU bringup */ > -unsigned long __initdata boot_gate = 1; > -/* Number of non-boot CPUs ready to enter C */ > -unsigned long __initdata ready_cpus = 0; > - > static __attribute_used__ void init_done(void) > { > free_init_memory(); > @@ -152,8 +147,6 @@ void __init start_xen(unsigned long boot > void *fdt; > size_t fdt_size; > int cpus, i; > - paddr_t gate_pa; > - unsigned long *gate; > > fdt = (void *)BOOT_MISC_VIRT_START > + (atag_paddr & ((1 << SECOND_SHIFT) - 1)); > @@ -169,25 +162,11 @@ void __init start_xen(unsigned long boot > console_init_preirq(); > #endif > > + cpus = gic_init(); > + make_cpus_ready(cpus, boot_phys_offset); > + > percpu_init_areas(); > set_processor_id(0); /* needed early, for smp_processor_id() */ > - > - cpus = gic_init(); > - > - printk("Waiting for %i other CPUs to be ready\n", cpus - 1); > - /* Bring the other CPUs up to paging before the original > - * copy of .text gets overwritten. We need to use the unrelocated > - * copy of boot_gate as that''s the one the others can see. */ > - gate_pa = ((unsigned long) &boot_gate) + boot_phys_offset; > - gate = map_domain_page(gate_pa >> PAGE_SHIFT) + (gate_pa & ~PAGE_MASK); > - *gate = 0; > - unmap_domain_page(gate); > - /* Now send an event to wake the first non-boot CPU */ > - asm volatile("dsb; isb; sev"); > - /* And wait for them all to be ready. */ > - while ( ready_cpus + 1 < cpus ) > - smp_rmb(); > - > __set_current((struct vcpu *)0xfffff000); /* debug sanity */ > idle_vcpu[0] = current; > > diff -r 4da1453ed61c -r 6f52e5de1b94 xen/arch/arm/smpboot.c > --- a/xen/arch/arm/smpboot.c Thu Mar 15 11:47:27 2012 +0000 > +++ b/xen/arch/arm/smpboot.c Thu Mar 15 14:13:57 2012 +0000 > @@ -19,6 +19,7 @@ > #include <xen/cpu.h> > #include <xen/cpumask.h> > #include <xen/delay.h> > +#include <xen/domain_page.h> > #include <xen/errno.h> > #include <xen/init.h> > #include <xen/mm.h> > @@ -41,11 +42,17 @@ static unsigned char __initdata cpu0_boo > /* Pointer to the stack, used by head.S when entering C */ > unsigned char *init_stack = cpu0_boot_stack; > > +/* Shared state for coordinating CPU bringup */ > +unsigned long smp_up_cpu = 0; > +static bool_t cpu_is_dead = 0; > + > +/* Number of non-boot CPUs ready to enter C */ > +unsigned long __initdata ready_cpus = 0; > + > void __init > smp_prepare_cpus (unsigned int max_cpus) > { > int i; > - set_processor_id(0); /* needed early, for smp_processor_id() */ > > cpumask_clear(&cpu_online_map); > cpumask_set_cpu(0, &cpu_online_map); > @@ -56,9 +63,30 @@ smp_prepare_cpus (unsigned int max_cpus) > cpumask_copy(&cpu_present_map, &cpu_possible_map); > } > > -/* Shared state for coordinating CPU bringup */ > -unsigned long smp_up_cpu = 0; > -static bool_t cpu_is_dead = 0; > +void __init > +make_cpus_ready(unsigned int max_cpus, unsigned long boot_phys_offset) > +{ > + unsigned long *gate; > + paddr_t gate_pa; > + int i; > + > + printk("Waiting for %i other CPUs to be ready\n", max_cpus - 1); > + /* We use the unrelocated copy of smp_up_cpu as that''s the one the > + * others can see. */ > + gate_pa = ((paddr_t) (unsigned long) &smp_up_cpu) + boot_phys_offset; > + gate = map_domain_page(gate_pa >> PAGE_SHIFT) + (gate_pa & ~PAGE_MASK); > + for ( i = 1; i < max_cpus; i++ ) > + { > + /* Tell the next CPU to get ready */ > + /* TODO: handle boards where CPUIDs are not contiguous */ > + *gate = i; > + asm volatile("dsb; isb; sev"); > + /* And wait for it to respond */ > + while ( ready_cpus < i ) > + smp_rmb(); > + } > + unmap_domain_page(gate); > +} > > /* Boot the current CPU */ > void __cpuinit start_secondary(unsigned long boot_phys_offset, > diff -r 4da1453ed61c -r 6f52e5de1b94 xen/include/asm-arm/smp.h > --- a/xen/include/asm-arm/smp.h Thu Mar 15 11:47:27 2012 +0000 > +++ b/xen/include/asm-arm/smp.h Thu Mar 15 14:13:57 2012 +0000 > @@ -16,6 +16,12 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_core_ > > extern void stop_cpu(void); > > +/* Bring the non-boot CPUs up to paging and ready to enter C. > + * Must be called after Xen is relocated but before the original copy of > + * .text gets overwritten. */ > +extern void > +make_cpus_ready(unsigned int max_cpus, unsigned long boot_phys_offset); > + > #endif > /* > * Local variables: