search for: interrupt_return

I'm implementing a more efficient version of the Xen iret paravirt_op, so that it can use the real iret instruction where possible. I really need to get access to per-cpu variables, so I can set the event mask state in the vcpu_info structure, but unfortunately at the point where INTERRUPT_RETURN is used in entry.S, the usermode %fs has already been restored. How would you feel if we changed paravirt_ops.iret to make it also responsible for restoring %fs? In other words, change RESTORE_REGS to skip %fs, and then native_iret would look like: 1: popl %fs iret with the normal exception s...

I'm implementing a more efficient version of the Xen iret paravirt_op, so that it can use the real iret instruction where possible. I really need to get access to per-cpu variables, so I can set the event mask state in the vcpu_info structure, but unfortunately at the point where INTERRUPT_RETURN is used in entry.S, the usermode %fs has already been restored. How would you feel if we changed paravirt_ops.iret to make it also responsible for restoring %fs? In other words, change RESTORE_REGS to skip %fs, and then native_iret would look like: 1: popl %fs iret with the normal exception s...

...l_fault 1: movl (%ebp),%ebp -.section __ex_table,"a" +.pushsection __ex_table,"a" .align 4 .long 1b,syscall_fault -.previous +.popsection = pushl %eax CFI_ADJUST_CFA_OFFSET 4 @@ -378,18 +378,18 @@ restore_nocheck_notrace: addl $4, %esp CFI_ADJUST_CFA_OFFSET -4 1: INTERRUPT_RETURN -.section .fixup,"ax" +.pushsection .fixup,"ax" iret_exc: TRACE_IRQS_ON ENABLE_INTERRUPTS pushl $0 # no error code pushl $do_iret_error jmp error_code -.previous -.section __ex_table,"a" +.popsection +.pushsection __ex_table,"a" .align 4 .lon...

...l_fault 1: movl (%ebp),%ebp -.section __ex_table,"a" +.pushsection __ex_table,"a" .align 4 .long 1b,syscall_fault -.previous +.popsection = pushl %eax CFI_ADJUST_CFA_OFFSET 4 @@ -378,18 +378,18 @@ restore_nocheck_notrace: addl $4, %esp CFI_ADJUST_CFA_OFFSET -4 1: INTERRUPT_RETURN -.section .fixup,"ax" +.pushsection .fixup,"ax" iret_exc: TRACE_IRQS_ON ENABLE_INTERRUPTS pushl $0 # no error code pushl $do_iret_error jmp error_code -.previous -.section __ex_table,"a" +.popsection +.pushsection __ex_table,"a" .align 4 .lon...

On Fri, Aug 07, 2020 at 10:38:23AM +0200, Juergen Gross wrote: > -# else > - const unsigned char cpu_iret[1]; > -# endif > }; > > static const struct patch_xxl patch_data_xxl = { > @@ -42,7 +38,6 @@ static const struct patch_xxl patch_data_xxl = { > .irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax > .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2,

On Fri, Aug 07, 2020 at 10:38:23AM +0200, Juergen Gross wrote: > -# else > - const unsigned char cpu_iret[1]; > -# endif > }; > > static const struct patch_xxl patch_data_xxl = { > @@ -42,7 +38,6 @@ static const struct patch_xxl patch_data_xxl = { > .irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax > .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2,

...3 >> - .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf >> - .cpu_iret = { 0xcf }, // iret >> -# endif > > I was looking at x86_64 paravirt the other day and found we actually > have pv_ops.cpu.iret users there.. On x86_64 we have (without PARAVIRT_XXL): #define INTERRUPT_RETURN jmp native_iret and with PARAVIRT_XXL this is basically a jmp *pv_ops.cpu.iret which will then be patched to either jmp native_iret or jmp xen_iret. On x86_32 INTERRUPT_RETURN was just "iret" for the non-paravirt case. This is the reason for above dropping of the static patch dat...

...LOCAL(early_idt_handler_common) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during very early boot. The + * early_idt_handler_array can't be used because it returns via the + * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. + */ +SYM_CODE_START_NOALIGN(vc_no_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /*...

...orig/arch/x86_64/kernel/entry.S +++ clean-start/arch/x86_64/kernel/entry.S @@ -51,6 +51,15 @@ #include <asm/page.h> #include <asm/irqflags.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ENABLE_INTERRUPTS(x) sti +#define DISABLE_INTERRUPTS(x) cli +#define INTERRUPT_RETURN iretq +#define SYSRETQ sysretq +#define SWAPGS swapgs +#endif .code64 #ifndef CONFIG_PREEMPT @@ -179,6 +188,7 @@ rff_trace: CFI_ENDPROC END(ret_from_fork) + /* * System call entry. Upto 6 arguments in registers are supported. * @@ -223,7 +233,7 @@ ENTRY(system_call) * No need...

...LOCAL(early_idt_handler_common) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during very early boot. The + * early_idt_handler_array can't be used because it returns via the + * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. + */ +SYM_CODE_START_NOALIGN(vc_no_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /*...

...orig/arch/x86_64/kernel/entry.S +++ clean-start/arch/x86_64/kernel/entry.S @@ -51,6 +51,15 @@ #include <asm/page.h> #include <asm/irqflags.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ENABLE_INTERRUPTS(x) sti +#define DISABLE_INTERRUPTS(x) cli +#define INTERRUPT_RETURN iretq +#define SYSRETQ sysretq +#define SWAPGS swapgs +#endif .code64 #ifndef CONFIG_PREEMPT @@ -179,6 +188,7 @@ rff_trace: CFI_ENDPROC END(ret_from_fork) + /* * System call entry. Upto 6 arguments in registers are supported. * @@ -223,7 +233,7 @@ ENTRY(system_call) * No need...

...addq $(6*8), %rsp @@ -1479,7 +1485,9 @@ first_nmi: addq $8, (%rsp) /* Fix up RSP */ pushfq /* RFLAGS */ pushq $__KERNEL_CS /* CS */ - pushq $1f /* RIP */ + pushq %rax /* Support Position Independent Code */ + leaq 1f(%rip), %rax /* RIP */ + xchgq %rax, (%rsp) /* Restore RAX, put 1f */ INTERRUPT_RETURN /* continues at repeat_nmi below */ UNWIND_HINT_IRET_REGS 1: -- 2.15.0.rc0.271.g36b669edcc-goog

..., paravirt_ops, irq_enable_sysexit); + OFFSET(PARAVIRT_iret, paravirt_ops, iret); + OFFSET(PARAVIRT_read_cr0, paravirt_ops, read_cr0); +#endif } =================================================================== --- a/arch/i386/kernel/entry.S +++ b/arch/i386/kernel/entry.S @@ -803,6 +803,15 @@ 1: INTERRUPT_RETURN .long 1b,iret_exc .previous +#ifdef CONFIG_PARAVIRT +ENTRY(nopara_iret) + iret + +ENTRY(nopara_irq_enable_sysexit) + sti + sysexit +#endif + KPROBE_ENTRY(int3) RING0_INT_FRAME pushl $-1 # mark this as an int =================================================================== --- a/inclu...

.../entry.S +++ b/arch/i386/kernel/entry.S @@ -76,13 +76,6 @@ NT_MASK = 0x00004000 NT_MASK = 0x00004000 VM_MASK = 0x00020000 -/* These are replaces for paravirtualization */ -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit -#define INTERRUPT_RETURN iret -#define GET_CR0_INTO_EAX movl %cr0, %eax - #ifdef CONFIG_PREEMPT #define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF #else @@ -809,6 +802,19 @@ 1: INTERRUPT_RETURN .long 1b,iret_exc .previous +#ifdef CONFIG_PARAVIRT +ENTRY(nopara_iret) +1: iret +.section __ex_table,"a&quot...

.../entry.S +++ b/arch/i386/kernel/entry.S @@ -76,13 +76,6 @@ NT_MASK = 0x00004000 NT_MASK = 0x00004000 VM_MASK = 0x00020000 -/* These are replaces for paravirtualization */ -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit -#define INTERRUPT_RETURN iret -#define GET_CR0_INTO_EAX movl %cr0, %eax - #ifdef CONFIG_PREEMPT #define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF #else @@ -809,6 +802,19 @@ 1: INTERRUPT_RETURN .long 1b,iret_exc .previous +#ifdef CONFIG_PARAVIRT +ENTRY(nopara_iret) +1: iret +.section __ex_table,"a&quot...

...CX, ecx); \ + COND_POP(set, CLBR_EDI, edi); \ + COND_POP(set, CLBR_EAX, eax) + +#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) +#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) +#define PARA_INDIRECT(addr) *%cs:addr +#endif + +#define INTERRUPT_RETURN \ + PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ + jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret)) + +#define DISABLE_INTERRUPTS(clobbers) \ + PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ + PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ + call PARA_IN...

...86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -136,10 +136,6 @@ static inline notrace unsigned long arch_local_irq_save(void) #define USERGS_SYSRET32 \ swapgs; \ sysretl -#define ENABLE_INTERRUPTS_SYSEXIT32 \ - swapgs; \ - sti; \ - sysexit #else #define INTERRUPT_RETURN iret diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 965c47d254aa..5f6051d5d139 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -976,11 +976,6 @@ extern void default_banner(void); PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_u...

...@ first_nmi: > addq $8, (%rsp) /* Fix up RSP */ > pushfq /* RFLAGS */ > pushq $__KERNEL_CS /* CS */ > - pushq $1f /* RIP */ > + pushq %rax /* Support Position Independent Code */ > + leaq 1f(%rip), %rax /* RIP */ > + xchgq %rax, (%rsp) /* Restore RAX, put 1f */ > INTERRUPT_RETURN /* continues at repeat_nmi below */ > UNWIND_HINT_IRET_REGS This patch seems to add extra overhead to the syscall fast-path even when PIE is disabled, right? Thanks, Ingo

...86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -136,10 +136,6 @@ static inline notrace unsigned long arch_local_irq_save(void) #define USERGS_SYSRET32 \ swapgs; \ sysretl -#define ENABLE_INTERRUPTS_SYSEXIT32 \ - swapgs; \ - sti; \ - sysexit #else #define INTERRUPT_RETURN iret diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 965c47d254aa..5f6051d5d139 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -976,11 +976,6 @@ extern void default_banner(void); PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_u...

Hi Andrew, This series of patches lays the basic ground work for the paravirtualized kernel patches coming later on. I think this lot is ready for the rough-and-tumble world of the -mm tree. The main change from the last posting is that all the page-table related patches have been moved out, and will be posted separately. Also, the off-by-one in reserving the top of address space has been