Hello all,
Here's a new version of the paravirt_ops x86_64 patch. With this
message, I'm sending an incremental patch. The complete patches can be
found , from now on, at http://et.redhat.com/~gcosta/paravirt_ops/
The main aim of this new update, is to fix a critical bug, namely,
Rusty's name. However, I took the opportunity to write some new less
important pieces of code, highlighting:
* proper casts in places in which macros were replaced by functions, and
the arguments happened to mismatch types.
* calling paravirt_ops functions from .S files (I lacked this last time)
* addition of the startup_paravirt function, to kick off guests (not
tested)
* fixed problems with patching
* added a new field, vsyscall_page in the paravirt_ops struct, which
allows the kernel to map a vsyscall_page on its own
* fixed vsyscall functions to avoid calling paravirt_ops functions.
__vsyscall_0 is the page to be mapped for the host. (set and get cpu not
yet tested.)
* fixed cpuid calls.
* added substitute for the swapgs instruction. (Notice that I'm not
saying it works ;-) )
In my TODO list, you can find:
* putting swapgs to work
* making sure legacy mode binaries work
* merging in valuable commentaries from all you ;-)
--
Glauber de Oliveira Costa
Red Hat Inc.
"Free as in Freedom"
-------------- next part --------------
diff -urp linux-2.6.19-paravirt0/arch/i386/kernel/alternative.c
linux-2.6.19-paravirt1/arch/i386/kernel/alternative.c
--- linux-2.6.19-paravirt0/arch/i386/kernel/alternative.c 2007-01-11
21:57:07.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/i386/kernel/alternative.c 2007-01-11
21:42:22.000000000 -0200
@@ -431,9 +431,7 @@ void __init alternative_instructions(voi
}
#endif
#ifdef CONFIG_PARAVIRT
- #ifndef CONFIG_X86_64 /* Not working properly yet */
apply_paravirt(__start_parainstructions, __stop_parainstructions);
- #endif
#endif
local_irq_restore(flags);
}
diff -urp linux-2.6.19-paravirt0/arch/x86_64/ia32/syscall32.c
linux-2.6.19-paravirt1/arch/x86_64/ia32/syscall32.c
--- linux-2.6.19-paravirt0/arch/x86_64/ia32/syscall32.c 2007-01-11
21:51:35.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/ia32/syscall32.c 2007-01-09
11:01:19.000000000 -0200
@@ -104,5 +104,5 @@ void syscall32_cpu_init(void)
checking_wrmsrl(MSR_IA32_SYSENTER_ESP, 0ULL);
checking_wrmsrl(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
- wrmsrl(MSR_CSTAR, ia32_cstar_target);
+ wrmsrl(MSR_CSTAR, (u64)ia32_cstar_target);
}
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/asm-offsets.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/asm-offsets.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/asm-offsets.c 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/asm-offsets.c 2007-01-11
09:46:44.000000000 -0200
@@ -79,9 +79,10 @@ int main(void)
ENTRY(paravirt_enabled);
ENTRY(irq_disable);
ENTRY(irq_enable);
- ENTRY(irq_enable_sysexit);
+ ENTRY(sysret);
ENTRY(iret);
- ENTRY(read_cr0);
+ ENTRY(read_cr2);
+ ENTRY(swapgs);
#endif
return 0;
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/entry.S
linux-2.6.19-paravirt1/arch/x86_64/kernel/entry.S
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/entry.S 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/entry.S 2007-01-11
22:22:26.000000000 -0200
@@ -51,6 +51,13 @@
#include <asm/page.h>
#include <asm/irqflags.h>
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define ENABLE_INTERRUPTS(x) sti
+#define DISABLE_INTERRUPTS(x) cli
+#define SYSRETQ sysretq
+#endif
.code64
#ifndef CONFIG_PREEMPT
@@ -179,6 +186,7 @@ rff_trace:
CFI_ENDPROC
END(ret_from_fork)
+
/*
* System call entry. Upto 6 arguments in registers are supported.
*
@@ -223,7 +231,7 @@ ENTRY(system_call)
* No need to follow this irqs off/on section - it's straight
* and short:
*/
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
SAVE_ARGS 8,1
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
movq %rcx,RIP-ARGOFFSET(%rsp)
@@ -245,7 +253,7 @@ ret_from_sys_call:
/* edi: flagmask */
sysret_check:
GET_THREAD_INFO(%rcx)
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
movl threadinfo_flags(%rcx),%edx
andl %edi,%edx
@@ -261,7 +269,7 @@ sysret_check:
/*CFI_REGISTER rflags,r11*/
movq %gs:pda_oldrsp,%rsp
swapgs
- sysretq
+ SYSRETQ
CFI_RESTORE_STATE
/* Handle reschedules */
@@ -270,7 +278,7 @@ sysret_careful:
bt $TIF_NEED_RESCHED,%edx
jnc sysret_signal
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule
@@ -281,7 +289,7 @@ sysret_careful:
/* Handle a signal */
sysret_signal:
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
jz 1f
@@ -294,7 +302,7 @@ sysret_signal:
1: movl $_TIF_NEED_RESCHED,%edi
/* Use IRET because user could have changed frame. This
works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
jmp int_with_check
@@ -326,7 +334,7 @@ tracesys:
*/
.globl int_ret_from_sys_call
int_ret_from_sys_call:
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
testl $3,CS-ARGOFFSET(%rsp)
je retint_restore_args
@@ -347,20 +355,20 @@ int_careful:
bt $TIF_NEED_RESCHED,%edx
jnc int_very_careful
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule
popq %rdi
CFI_ADJUST_CFA_OFFSET -8
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
jmp int_with_check
/* handle signals and tracing -- both require a full stack frame */
int_very_careful:
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
SAVE_REST
/* Check for syscall exit trace */
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -383,7 +391,7 @@ int_signal:
1: movl $_TIF_NEED_RESCHED,%edi
int_restore_rest:
RESTORE_REST
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
jmp int_with_check
CFI_ENDPROC
@@ -525,7 +533,7 @@ ENTRY(common_interrupt)
interrupt do_IRQ
/* 0(%rsp): oldrsp-ARGOFFSET */
ret_from_intr:
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
decl %gs:pda_irqcount
leaveq
@@ -552,13 +560,13 @@ retint_swapgs:
/*
* The iretq could re-enable interrupts:
*/
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_IRETQ
swapgs
jmp restore_args
retint_restore_args:
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
/*
* The iretq could re-enable interrupts:
*/
@@ -566,35 +574,22 @@ retint_restore_args:
restore_args:
RESTORE_ARGS 0,8,0
iret_label:
- iretq
+ INTERRUPT_RETURN
- .section __ex_table,"a"
- .quad iret_label,bad_iret
- .previous
- .section .fixup,"ax"
- /* force a signal here? this matches i386 behaviour */
- /* running with kernel gs */
-bad_iret:
- movq $11,%rdi /* SIGSEGV */
- TRACE_IRQS_ON
- sti
- jmp do_exit
- .previous
-
/* edi: workmask, edx: work */
retint_careful:
CFI_RESTORE_STATE
bt $TIF_NEED_RESCHED,%edx
jnc retint_signal
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule
popq %rdi
CFI_ADJUST_CFA_OFFSET -8
GET_THREAD_INFO(%rcx)
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
jmp retint_check
@@ -602,14 +597,14 @@ retint_signal:
testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
jz retint_swapgs
TRACE_IRQS_ON
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
SAVE_REST
movq $-1,ORIG_RAX(%rsp)
xorl %esi,%esi # oldset
movq %rsp,%rdi # &pt_regs
call do_notify_resume
RESTORE_REST
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
movl $_TIF_NEED_RESCHED,%edi
GET_THREAD_INFO(%rcx)
@@ -738,7 +733,7 @@ END(spurious_interrupt)
.if \ist
addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
.endif
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
.if \irqtrace
TRACE_IRQS_OFF
.endif
@@ -770,7 +765,7 @@ paranoid_swapgs\trace:
swapgs
paranoid_restore\trace:
RESTORE_ALL 8
- iretq
+ INTERRUPT_RETURN
paranoid_userspace\trace:
GET_THREAD_INFO(%rcx)
movl threadinfo_flags(%rcx),%ebx
@@ -785,11 +780,11 @@ paranoid_userspace\trace:
.if \trace
TRACE_IRQS_ON
.endif
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
xorl %esi,%esi /* arg2: oldset */
movq %rsp,%rdi /* arg1: &pt_regs */
call do_notify_resume
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
.if \trace
TRACE_IRQS_OFF
.endif
@@ -798,9 +793,9 @@ paranoid_schedule\trace:
.if \trace
TRACE_IRQS_ON
.endif
- sti
+ ENABLE_INTERRUPTS(CLBR_NONE)
call schedule
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
.if \trace
TRACE_IRQS_OFF
.endif
@@ -862,7 +857,7 @@ error_sti:
error_exit:
movl %ebx,%eax
RESTORE_REST
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
GET_THREAD_INFO(%rcx)
testl %eax,%eax
@@ -904,7 +899,7 @@ ENTRY(load_gs_index)
CFI_STARTPROC
pushf
CFI_ADJUST_CFA_OFFSET 8
- cli
+ DISABLE_INTERRUPTS(CLBR_NONE)
swapgs
gs_change:
movl %edi,%gs
@@ -1065,18 +1060,32 @@ KPROBE_ENTRY(int3)
KPROBE_END(int3)
#ifdef CONFIG_PARAVIRT
+/* Not yet working. Do not use */
+ENTRY(native_swapgs)
+ swapgs
+ jmp %cs:(paravirt_ops+PARAVIRT_swapgs)
+ENDPROC(native_swapgs)
+
ENTRY(native_iret)
1: iretq
.section __ex_table,"a"
.align 8
.quad 1b, bad_iret
.previous
+.section .fixup,"ax"
+/* force a signal here? this matches i386 behaviour */
+/* running with kernel gs */
+bad_iret:
+ movq $11,%rdi /* SIGSEGV */
+ TRACE_IRQS_ON
+ ENABLE_INTERRUPTS(CLBR_NONE)
+ jmp do_exit
+ .previous
ENDPROC(native_iret)
-ENTRY(native_irq_enable_sysexit)
- sti
+ENTRY(native_sysret)
sysretq
-ENDPROC(native_irq_enable_sysexit)
+ENDPROC(native_sysret)
#endif /* CONFIG_PARAVIRT */
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/head64.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/head64.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/head64.c 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/head64.c 2007-01-09
18:13:19.000000000 -0200
@@ -62,7 +62,7 @@ void __init x86_64_start_kernel(char * r
for (i = 0; i < IDT_ENTRIES; i++)
set_intr_gate(i, early_idt_handler);
- asm volatile("lidt %0" :: "m" (idt_descr));
+ load_idt((const struct desc_struct *)&idt_descr);
early_printk("Kernel alive\n");
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/head.S
linux-2.6.19-paravirt1/arch/x86_64/kernel/head.S
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/head.S 2006-12-11
17:32:53.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/head.S 2007-01-11
22:42:33.000000000 -0200
@@ -16,6 +16,13 @@
#include <asm/page.h>
#include <asm/msr.h>
#include <asm/cache.h>
+
+#ifdef CONFIG_PARAVIRT
+#include <asm/asm-offsets.h>
+#include <asm/paravirt.h>
+#else
+#define GET_CR2_INTO_RAX mov %cr2, %rax
+#endif
/* we are not able to switch in one step to the final KERNEL ADRESS SPACE
* because we need identity-mapped pages on setup so define __START_KERNEL to
@@ -106,6 +113,14 @@ startup_64:
* reload the page tables here.
*/
+#ifdef CONFIG_PARAVIRT
+ /* a CS ended in 0x3 indicates we're in userspace. That's where
+ * our paravirt guests run. */
+ movq %cs, %rax
+ testq $0x3, %rax
+ jnz startup_paravirt
+#endif
+
/* Enable PAE mode and PGE */
xorq %rax, %rax
btsq $5, %rax
@@ -208,10 +223,11 @@ ENTRY(early_idt_handler)
cmpl $2,early_recursion_flag(%rip)
jz 1f
incl early_recursion_flag(%rip)
- xorl %eax,%eax
movq 8(%rsp),%rsi # get rip
movq (%rsp),%rdx
- movq %cr2,%rcx
+ GET_CR2_INTO_RAX
+ movq %rax,%rcx
+ xorq %rax, %rax
leaq early_idt_msg(%rip),%rdi
call early_printk
cmpl $2,early_recursion_flag(%rip)
@@ -232,6 +248,47 @@ early_idt_msg:
early_idt_ripmsg:
.asciz "RIP %s\n"
+#ifdef CONFIG_PARAVIRT
+ENTRY(startup_paravirt)
+ cld
+
+ /* initial stack location */
+ movq $(init_thread_union+THREAD_SIZE),%rsp
+
+ /* We take pains to preserve all the regs. */
+ pushq %r11
+ pushq %r10
+ pushq %r9
+ pushq %r8
+ pushq %rsi
+ pushq %rdi
+ pushq %rdx
+ pushq %rcx
+ pushq %rax
+
+ /* paravirt.o is last in link, and that probe fn never returns */
+ pushq $__start_paravirtprobe
+1:
+ movq 0(%rsp), %rax
+ pushq (%rax)
+ movq 8(%rsp), %rdi
+ call *(%rsp)
+ popq %rax
+
+ movq 0x10(%rsp), %rax
+ movq 0x18(%rsp), %rcx
+ movq 0x20(%rsp), %rdx
+ movq 0x28(%rsp), %rdi
+ movq 0x30(%rsp), %rsi
+ movq 0x38(%rsp), %r8
+ movq 0x40(%rsp), %r9
+ movq 0x48(%rsp), %r10
+ movq 0x50(%rsp), %r11
+
+ addl $8, (%rsp)
+ jmp 1b
+#endif
+
.code32
ENTRY(no_long_mode)
/* This isn't an x86-64 CPU so hang */
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/paravirt.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/paravirt.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/paravirt.c 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/paravirt.c 2007-01-11
20:10:06.000000000 -0200
@@ -1,6 +1,6 @@
/* Paravirtualization interfaces
Copyright (C) 2007 Glauber de Oliveira Costa, Red Hat Inc.
- Based on i386 work by Rusty Russel.
+ Based on i386 work by Rusty Russell.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -59,11 +59,14 @@ void memory_setup(void)
asm("start_" #name ": " code "; end_" #name
":")
DEF_NATIVE(cli, "cli");
DEF_NATIVE(sti, "sti");
-DEF_NATIVE(popfq, "pushq %rax; popfq");
+/* We push rdi , and pop in rda. This is due to x86_64 calling conventions
+ * Recall that we are patching a function call */
+DEF_NATIVE(popfq, "pushq %rdi; popfq");
DEF_NATIVE(pushfq, "pushfq; popq %rax");
DEF_NATIVE(pushfq_cli, "pushfq; popq %rax; cli");
-DEF_NATIVE(iret, "iret");
-DEF_NATIVE(sti_sysretq, "sti; sysretq");
+DEF_NATIVE(iret, "iretq");
+DEF_NATIVE(sysretq, "sysretq");
+DEF_NATIVE(swapgs, "swapgs");
static const struct native_insns
{
@@ -75,7 +78,8 @@ static const struct native_insns
[PARAVIRT_SAVE_FLAGS] = { start_pushfq, end_pushfq },
[PARAVIRT_SAVE_FLAGS_IRQ_DISABLE] = { start_pushfq_cli, end_pushfq_cli },
[PARAVIRT_INTERRUPT_RETURN] = { start_iret, end_iret },
- [PARAVIRT_STI_SYSRETQ] = { start_sti_sysretq, end_sti_sysretq },
+ [PARAVIRT_SYSRETQ] = { start_sysretq, end_sysretq },
+ [PARAVIRT_SWAPGS] = { start_swapgs, end_swapgs },
};
static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)
@@ -88,7 +92,6 @@ static unsigned native_patch(u8 type, u1
insn_len = native_insns[type].end - native_insns[type].start;
-
/* Similarly if we can't fit replacement. */
if (len < insn_len)
return len;
@@ -243,7 +246,7 @@ static void native_wbinvd(void)
asm volatile("wbinvd": : :"memory");
}
-static unsigned long native_read_msr(unsigned int msr, int *err)
+static u64 native_read_msr(unsigned int msr, int *err)
{
unsigned long val;
@@ -287,6 +290,13 @@ static u64 native_read_tsc(void)
return val;
}
+static u64 native_read_tscp(int *aux)
+{
+ u64 val;
+ asm volatile ("rdtscp" : "=A" (val), "=c"
(aux));
+ return val;
+}
+
static u64 native_read_pmc(void)
{
unsigned long val;
@@ -463,7 +473,8 @@ void native_pmd_clear(pmd_t *pmd)
/* These are in entry.S */
extern void native_iret(void);
-extern void native_irq_enable_sysexit(void);
+extern void native_sysret(void);
+extern void native_swapgs(void);
static int __init print_banner(void)
{
@@ -475,12 +486,18 @@ core_initcall(print_banner);
/* We simply declare start_kernel to be the paravirt probe of last resort. */
paravirt_probe(start_kernel);
+extern unsigned long __vsyscall_0;
struct paravirt_ops paravirt_ops = {
.name = "bare hardware",
.paravirt_enabled = 0,
.kernel_rpl = 0,
.pgd_alignment = sizeof(pgd_t) * PTRS_PER_PGD,
+ .swapgs = {
+ .ret = 0,
+ .fn = native_swapgs,
+ },
+ .vsyscall_page = &__vsyscall_0,
.patch = native_patch,
.banner = default_banner,
.arch_setup = native_nop,
@@ -512,6 +529,7 @@ struct paravirt_ops paravirt_ops = {
.read_msr = native_read_msr,
.write_msr = native_write_msr,
.read_tsc = native_read_tsc,
+ .read_tscp = native_read_tscp,
.read_pmc = native_read_pmc,
.load_tr_desc = native_load_tr_desc,
.set_ldt = native_set_ldt,
@@ -571,7 +589,7 @@ struct paravirt_ops paravirt_ops = {
.make_pud = native_make_pud,
.make_pgd = native_make_pgd,
- .irq_enable_sysexit = native_irq_enable_sysexit,
+ .sysret = native_sysret,
.iret = native_iret,
.dup_mmap = (void *)native_nop,
@@ -580,4 +598,5 @@ struct paravirt_ops paravirt_ops = {
.startup_ipi_hook = (void *)native_nop,
};
+
EXPORT_SYMBOL(paravirt_ops);
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/setup64.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/setup64.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/setup64.c 2006-12-11
17:32:53.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/setup64.c 2007-01-09
10:24:25.000000000 -0200
@@ -123,7 +123,7 @@ void pda_init(int cpu)
asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
/* Memory clobbers used to order PDA accessed */
mb();
- wrmsrl(MSR_GS_BASE, pda);
+ wrmsrl(MSR_GS_BASE, (u64)pda);
mb();
pda->cpunumber = cpu;
@@ -160,7 +160,7 @@ void syscall_init(void)
* but only a 32bit target. LSTAR sets the 64bit rip.
*/
wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 |
((u64)__KERNEL_CS)<<32);
- wrmsrl(MSR_LSTAR, system_call);
+ wrmsrl(MSR_LSTAR, (u64)system_call);
#ifdef CONFIG_IA32_EMULATION
syscall32_cpu_init ();
@@ -223,8 +223,8 @@ void __cpuinit cpu_init (void)
memcpy(cpu_gdt(cpu), cpu_gdt_table, GDT_SIZE);
cpu_gdt_descr[cpu].size = GDT_SIZE;
- asm volatile("lgdt %0" :: "m" (cpu_gdt_descr[cpu]));
- asm volatile("lidt %0" :: "m" (idt_descr));
+ load_gdt((const struct desc_struct *)&cpu_gdt_descr[cpu]);
+ load_idt((const struct desc_struct *)&idt_descr);
memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
syscall_init();
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/setup.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/setup.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/setup.c 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/setup.c 2007-01-09
10:22:24.000000000 -0200
@@ -341,6 +341,12 @@ static void discover_ebda(void)
ebda_size = 64*1024;
}
+/* Overridden in paravirt.c if CONFIG_PARAVIRT */
+void __attribute__((weak)) memory_setup(void)
+{
+ return setup_memory_region();
+}
+
void __init setup_arch(char **cmdline_p)
{
printk(KERN_INFO "Command line: %s\n", saved_command_line);
@@ -561,12 +567,6 @@ static int __cpuinit get_model_name(stru
return 1;
}
-/* Overridden in paravirt.c if CONFIG_PARAVIRT */
-void __attribute__((weak)) memory_setup(void)
-{
- return setup_memory_region();
-}
-
static void __cpuinit display_cacheinfo(struct cpuinfo_x86 *c)
{
unsigned int n, dummy, eax, ebx, ecx, edx;
diff -urp linux-2.6.19-paravirt0/arch/x86_64/kernel/vsyscall.c
linux-2.6.19-paravirt1/arch/x86_64/kernel/vsyscall.c
--- linux-2.6.19-paravirt0/arch/x86_64/kernel/vsyscall.c 2007-01-11
21:51:35.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/kernel/vsyscall.c 2007-01-10
06:57:22.000000000 -0200
@@ -73,7 +73,7 @@ static __always_inline void do_vgettimeo
usec = __xtime.tv_nsec / 1000;
if (__vxtime.mode != VXTIME_HPET) {
- t = get_cycles_sync();
+ t = vget_cycles_sync();
if (t < __vxtime.last_tsc)
t = __vxtime.last_tsc;
usec += ((t - __vxtime.last_tsc) *
@@ -147,8 +147,8 @@ time_t __vsyscall(1) vtime(time_t *t)
long __vsyscall(2)
vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
{
- unsigned int dummy, p;
- unsigned long j = 0;
+ unsigned int p;
+ unsigned long dummy, j = 0;
/* Fast cache - only recompute value once per jiffies and avoid
relatively costly rdtscp/cpuid otherwise.
@@ -162,7 +162,8 @@ vgetcpu(unsigned *cpu, unsigned *node, s
p = tcache->blob[1];
} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
/* Load per CPU data from RDTSCP */
- rdtscp(dummy, dummy, p);
+ /* rdtscp() cannot be called due to the paravirt indirection */
+ asm("rdtscp" : "=A" (dummy), "=c" (p));
} else {
/* Load per CPU data from GDT */
asm("lsl %1,%0" : "=r" (p) : "r"
(__PER_CPU_SEG));
@@ -257,7 +258,11 @@ static void __cpuinit vsyscall_set_cpu(i
node = cpu_to_node[cpu];
#endif
if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
- write_rdtscp_aux((node << 12) | cpu);
+ /* This is write_rdtscp_aux. It cannot be called directly
+ * due to the paravirt indirection */
+ asm("wrmsr" : /* no output */
+ : "d"(0),
+ "a" ((node << 12) | cpu), "c" (0xc0000103));
/* Store cpu number in limit so that it can be loaded quickly
in user space in vgetcpu.
@@ -286,8 +291,12 @@ cpu_vsyscall_notifier(struct notifier_bl
static void __init map_vsyscall(void)
{
+#ifndef CONFIG_PARAVIRT
extern char __vsyscall_0;
unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
+#else
+ unsigned long physaddr_page0 = __pa_symbol(paravirt_ops.vsyscall_page);
+#endif
/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
@@ -300,7 +309,14 @@ static int __init vsyscall_init(void)
BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
- map_vsyscall();
+#ifdef CONFIG_PARAVIRT
+ if (paravirt_ops.vsyscall_page)
+#endif
+ map_vsyscall();
+#ifdef CONFIG_PARAVIRT
+ else
+ __sysctl_vsyscall = 0;
+#endif
#ifdef CONFIG_SYSCTL
register_sysctl_table(kernel_root_table2, 0);
#endif
diff -urp linux-2.6.19-paravirt0/arch/x86_64/mm/pageattr.c
linux-2.6.19-paravirt1/arch/x86_64/mm/pageattr.c
--- linux-2.6.19-paravirt0/arch/x86_64/mm/pageattr.c 2007-01-11
21:51:35.000000000 -0200
+++ linux-2.6.19-paravirt1/arch/x86_64/mm/pageattr.c 2007-01-09
18:02:50.000000000 -0200
@@ -81,7 +81,7 @@ static void flush_kernel_map(void *arg)
void *adr = page_address(pg);
if (cpu_has_clflush)
cache_flush_page(adr);
- __flush_tlb_one(adr);
+ __flush_tlb_one((u64)adr);
}
}
diff -urp linux-2.6.19-paravirt0/include/asm-x86_64/alternative.h
linux-2.6.19-paravirt1/include/asm-x86_64/alternative.h
--- linux-2.6.19-paravirt0/include/asm-x86_64/alternative.h 2007-01-11
21:51:36.000000000 -0200
+++ linux-2.6.19-paravirt1/include/asm-x86_64/alternative.h 2007-01-08
06:53:56.000000000 -0200
@@ -134,8 +134,10 @@ static inline void alternatives_smp_swit
#define LOCK_PREFIX ""
#endif
-struct paravirt_patch;
+
+
#ifdef CONFIG_PARAVIRT
+struct paravirt_patch;
void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end);
#else
static inline void
@@ -145,4 +147,5 @@ apply_paravirt(struct paravirt_patch *st
#define __stop_parainstructions NULL
#endif
+
#endif /* _X86_64_ALTERNATIVE_H */
diff -urp linux-2.6.19-paravirt0/include/asm-x86_64/irqflags.h
linux-2.6.19-paravirt1/include/asm-x86_64/irqflags.h
--- linux-2.6.19-paravirt0/include/asm-x86_64/irqflags.h 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/include/asm-x86_64/irqflags.h 2007-01-09
17:55:54.000000000 -0200
@@ -18,7 +18,6 @@ static inline int raw_irqs_disabled_flag
{
return !(flags & (1 << 9));
}
-
#else
/*
diff -urp linux-2.6.19-paravirt0/include/asm-x86_64/msr.h
linux-2.6.19-paravirt1/include/asm-x86_64/msr.h
--- linux-2.6.19-paravirt0/include/asm-x86_64/msr.h 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/include/asm-x86_64/msr.h 2007-01-09
18:12:03.000000000 -0200
@@ -105,15 +105,6 @@ static inline void native_cpuid(unsigned
#endif /* CONFIG_PARAVIRT */
-#define rdtscp(low,high,aux) \
- asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (low),
"=d" (high), "=c" (aux))
-
-#define rdtscpll(val, aux) do { \
- unsigned long __a, __d; \
- asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (__a),
"=d" (__d), "=c" (aux)); \
- (val) = (__d << 32) | __a; \
-} while (0)
-
#define checking_wrmsrl(msr,val)
wrmsr_safe(msr,(u32)(val),(u32)((val)>>32))
#define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
@@ -125,6 +116,7 @@ static inline void cpuid(unsigned int op
*eax = op;
__cpuid(eax, ebx, ecx, edx);
}
+
/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(int op, int count,
int *eax, int *ebx, int *ecx, int *edx)
@@ -140,24 +132,28 @@ static inline void cpuid_count(int op, i
static inline unsigned int cpuid_eax(unsigned int op)
{
unsigned int eax, ebx, ecx, edx;
+ eax = op;
__cpuid(&eax, &ebx, &ecx, &edx);
return eax;
}
static inline unsigned int cpuid_ebx(unsigned int op)
{
unsigned int eax, ebx, ecx, edx;
+ eax = op;
__cpuid(&eax, &ebx, &ecx, &edx);
return ebx;
}
static inline unsigned int cpuid_ecx(unsigned int op)
{
unsigned int eax, ebx, ecx, edx;
+ eax = op;
__cpuid(&eax, &ebx, &ecx, &edx);
return ecx;
}
static inline unsigned int cpuid_edx(unsigned int op)
{
unsigned int eax, ebx, ecx, edx;
+ eax = op;
__cpuid(&eax, &ebx, &ecx, &edx);
return edx;
}
diff -urp linux-2.6.19-paravirt0/include/asm-x86_64/paravirt.h
linux-2.6.19-paravirt1/include/asm-x86_64/paravirt.h
--- linux-2.6.19-paravirt0/include/asm-x86_64/paravirt.h 2007-01-11
21:56:03.000000000 -0200
+++ linux-2.6.19-paravirt1/include/asm-x86_64/paravirt.h 2007-01-11
22:50:41.000000000 -0200
@@ -17,7 +17,8 @@
#define PARAVIRT_SAVE_FLAGS 3
#define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
#define PARAVIRT_INTERRUPT_RETURN 5
-#define PARAVIRT_STI_SYSRETQ 6
+#define PARAVIRT_SYSRETQ 6
+#define PARAVIRT_SWAPGS 7
/* Bitmask of what can be clobbered: usually at least rax. */
#define CLBR_NONE 0x0
@@ -34,6 +35,11 @@ struct desc_struct;
struct tss_struct;
struct mm_struct;
+struct swapgs {
+ u64 ret;
+ void (*fn)(void);
+};
+
struct paravirt_ops
{
int paravirt_enabled;
@@ -43,6 +49,9 @@ struct paravirt_ops
const char *name;
+ unsigned long *vsyscall_page;
+
+ struct swapgs swapgs;
/*
* Patch may replace one of the defined code sequences with arbitrary
* code, subject to the same register constraints. This generally
@@ -89,6 +98,7 @@ struct paravirt_ops
void (*restore_fl)(unsigned long);
void (*irq_disable)(void);
void (*irq_enable)(void);
+
void (*safe_halt)(void);
void (*halt)(void);
void (*wbinvd)(void);
@@ -98,6 +108,7 @@ struct paravirt_ops
int (*write_msr)(unsigned int msr, u64 val);
u64 (*read_tsc)(void);
+ u64 (*read_tscp)(int *aux);
u64 (*read_pmc)(void);
void (*load_tr_desc)(void);
@@ -167,7 +178,7 @@ struct paravirt_ops
void (*set_lazy_mode)(int mode);
/* These two are jmp to, not actually called. */
- void (*irq_enable_sysexit)(void);
+ void (*sysret)(void);
void (*iret)(void);
void (*startup_ipi_hook)(int phys_apicid, unsigned long start_eip, unsigned
long start_esp);
@@ -262,6 +273,14 @@ static inline void halt(void)
val2 = _l >> 32; \
} while(0)
+/* rdmsr with exception handling */
+#define rdmsr_safe(msr,a,b) ({ \
+ int _err; \
+ u64 _l = paravirt_ops.read_msr(msr,&_err); \
+ (*a) = (u32)_l; \
+ (*b) = _l >> 32; \
+ _err; })
+
#define wrmsr(msr,val1,val2) do { \
u64 _l = ((u64)(val2) << 32) | (val1); \
paravirt_ops.write_msr((msr), _l); \
@@ -273,19 +292,12 @@ static inline void halt(void)
} while(0)
#define wrmsrl(msr,val) (paravirt_ops.write_msr((msr),(val)))
+
#define wrmsr_safe(msr,a,b) ({ \
u64 _l = ((u64)(b) << 32) | (a); \
paravirt_ops.write_msr((msr),_l); \
})
-/* rdmsr with exception handling */
-#define rdmsr_safe(msr,a,b) ({ \
- int _err; \
- u64 _l = paravirt_ops.read_msr(msr,&_err); \
- (*a) = (u32)_l; \
- (*b) = _l >> 32; \
- _err; })
-
#define rdtsc(low,high) do { \
u64 _l = paravirt_ops.read_tsc(); \
low = (u32)_l; \
@@ -299,6 +311,14 @@ static inline void halt(void)
#define rdtscll(val) (val = paravirt_ops.read_tsc())
+#define rdtscp(low,high,aux) do { \
+ u64 _val = paravirt_ops.read_tscp(&aux); \
+ low = (int)_val; \
+ high = _val >> 32; \
+} while (0)
+
+#define rdtscpll(val, aux) (val) = paravirt_ops.read_tscp(&aux)
+
#define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
#define rdpmc(counter,low,high) do { \
@@ -375,7 +395,6 @@ void native_pte_clear(struct mm_struct *
void native_pmd_clear(pmd_t *pmd);
void native_nop(void);
-
static inline void paravirt_activate_mm(struct mm_struct *prev,
struct mm_struct *next)
{
@@ -483,6 +502,9 @@ struct paravirt_patch {
" .short " __stringify(clobber) "\n" \
".popsection"
+/* These functions tends to be very simple. So, if they touch any register,
+ * the calle-saved ones may already fulfill their needs, and hopefully we
+ * have no need to save any. */
static inline unsigned long __raw_local_save_flags(void)
{
unsigned long f;
@@ -533,18 +555,12 @@ static inline unsigned long __raw_local_
return f;
}
+#define CLI_STRING paravirt_alt("call
*paravirt_ops+%c[irq_disable];", \
+ PARAVIRT_IRQ_DISABLE, CLBR_NONE)
+#define STI_STRING paravirt_alt("call *paravirt_ops+%c[irq_enable];",
\
+ PARAVIRT_IRQ_ENABLE, CLBR_NONE)
-/* Still x86-ish */
-#define CLI_STRING paravirt_alt("pushq %%rcx; pushq %%rdx;" \
- "call *paravirt_ops+%c[irq_disable];" \
- "popq %%rdx; popq %%rcx", \
- PARAVIRT_IRQ_DISABLE, CLBR_RAX)
-
-#define STI_STRING paravirt_alt("pushq %%rcx; pushq %%rdx;" \
- "call *paravirt_ops+%c[irq_enable];" \
- "popq %%rdx; popq %%rcx", \
- PARAVIRT_IRQ_ENABLE, CLBR_RAX)
#define CLI_STI_CLOBBERS , "%rax"
#define CLI_STI_INPUT_ARGS \
, \
@@ -571,22 +587,23 @@ static inline unsigned long __raw_local_
#define DISABLE_INTERRUPTS(clobbers) \
PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \
- pushq %rcx; pushq %rdx; \
- call *paravirt_ops+PARAVIRT_irq_disable; \
- popq %rdx; popq %rcx) \
+ call *paravirt_ops+PARAVIRT_irq_disable)
#define ENABLE_INTERRUPTS(clobbers) \
PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \
- pushq %rcx; pushq %rdx; \
- call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
- popq %rdx; popq %rcx)
-
-#define ENABLE_INTERRUPTS_SYSRETQ \
- PARA_PATCH(PARAVIRT_STI_SYSRETQ, CLBR_ANY, \
- jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
+ call *%cs:paravirt_ops+PARAVIRT_irq_enable)
-#define GET_CR0_INTO_RAX \
- call *paravirt_ops+PARAVIRT_read_cr0
+#define SYSRETQ \
+ PARA_PATCH(PARAVIRT_SYSRETQ, CLBR_ANY, \
+ jmp *%cs:paravirt_ops+PARAVIRT_sysret)
+
+#define SWAPGS \
+ movq $. + 0x11, (paravirt_ops+PARAVIRT_swapgs); \
+ jmp (paravirt_ops+PARAVIRT_swapgs+8); \
+
+/* this is needed in early_idt_handler */
+#define GET_CR2_INTO_RAX \
+ call *paravirt_ops+PARAVIRT_read_cr2
#endif /* __ASSEMBLY__ */
#else /* !CONFIG_PARAVIRT */
diff -urp linux-2.6.19-paravirt0/include/asm-x86_64/timex.h
linux-2.6.19-paravirt1/include/asm-x86_64/timex.h
--- linux-2.6.19-paravirt0/include/asm-x86_64/timex.h 2006-12-11
17:32:53.000000000 -0200
+++ linux-2.6.19-paravirt1/include/asm-x86_64/timex.h 2007-01-10
15:10:00.000000000 -0200
@@ -31,14 +31,29 @@ static __always_inline cycles_t get_cycl
{
unsigned long long ret;
unsigned eax;
+ unsigned int (*fn)(unsigned int) = &cpuid_eax;
/* Don't do an additional sync on CPUs where we know
RDTSC is already synchronous. */
- alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
- "=a" (eax), "0" (1) :
"ebx","ecx","edx","memory");
+ alternative_io("call *%3", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
+ "=a" (eax) , "D" (1) , "m" (fn));
rdtscll(ret);
return ret;
}
+/* Inside a vsyscall, we cannot call paravirt functions. (like rdtsc
+ * and cpuid). For the host, use this function instead */
+static __always_inline cycles_t vget_cycles_sync(void)
+{
+ unsigned long ret;
+ unsigned eax;
+ /* Don't do an additional sync on CPUs where we know
+ RDTSC is already synchronous. */
+ alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
+ "=a" (eax), "0" (1) :
"ebx","ecx","edx","memory");
+
+ asm volatile("rdtsc" : "=A" (ret));
+ return ret;
+}
extern unsigned int cpu_khz;
extern int read_current_timer(unsigned long *timer_value);
