search for: iretq

...kernel/entry.S +++ clean-start/arch/x86_64/kernel/entry.S @@ -51,6 +51,15 @@ #include <asm/page.h> #include <asm/irqflags.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ENABLE_INTERRUPTS(x) sti +#define DISABLE_INTERRUPTS(x) cli +#define INTERRUPT_RETURN iretq +#define SYSRETQ sysretq +#define SWAPGS swapgs +#endif .code64 #ifndef CONFIG_PREEMPT @@ -179,6 +188,7 @@ rff_trace: CFI_ENDPROC END(ret_from_fork) + /* * System call entry. Upto 6 arguments in registers are supported. * @@ -223,7 +233,7 @@ ENTRY(system_call) * No need to fol...

...kernel/entry.S +++ clean-start/arch/x86_64/kernel/entry.S @@ -51,6 +51,15 @@ #include <asm/page.h> #include <asm/irqflags.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ENABLE_INTERRUPTS(x) sti +#define DISABLE_INTERRUPTS(x) cli +#define INTERRUPT_RETURN iretq +#define SYSRETQ sysretq +#define SWAPGS swapgs +#endif .code64 #ifndef CONFIG_PREEMPT @@ -179,6 +188,7 @@ rff_trace: CFI_ENDPROC END(ret_from_fork) + /* * System call entry. Upto 6 arguments in registers are supported. * @@ -223,7 +233,7 @@ ENTRY(system_call) * No need to fol...

...V_ES_NMI_COMPLETE + /* * Return back to user mode. We must *not* do the normal exit * work, because we don't want to enable interrupts. @@ -1599,6 +1609,7 @@ nested_nmi_out: popq %rdx /* We are returning to kernel mode, so this cannot result in a fault. */ + SEV_ES_NMI_COMPLETE iretq first_nmi: @@ -1687,6 +1698,12 @@ end_repeat_nmi: movq $-1, %rsi call do_nmi + /* + * When running as an SEV-ES guest, jump to the SEV-ES NMI IRET + * path. + */ + SEV_ES_NMI_COMPLETE + /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg...

...e pushing */ + + /* Set up stack frame */ + pushq $0 /* SS */ + pushq %rax /* RSP */ + pushfq /* RFLAGS */ + pushq $__HYPERVISOR_CS /* CS */ + leaq 1f(%rip),%rax + pushq %rax /* RIP */ + + iretq /* Disable the hardware NMI latch */ +1: + retq + +/* No op trap handler. Required for kexec crash path. This is not + * declared with the ENTRY() macro to avoid wasted alignment space. + */ +.globl trap_nop +trap_nop: + iretq + + + .section .rodata, "a", @progbits ENT...

On 14/03/2018 16:54, Christopher Lameter wrote: >>> + pushq %rax /* Support Position Independent Code */ >>> + leaq 1f(%rip), %rax /* RIP */ >>> + xchgq %rax, (%rsp) /* Restore RAX, put 1f */ >>> iretq /* continues at repeat_nmi below */ >>> UNWIND_HINT_IRET_REGS >>> 1: >> Urgh, xchg with a memop has an implicit LOCK prefix. > this_cpu_xchg uses no lock cmpxchg as a replacement to reduce latency. That requires using a second register, since %rax is used as the com...

This set of patches converts the way frames gets created from using PUSHes/POPs to using MOVes, thus allowing (in certain cases) to avoid saving/restoring part of the register set. While the place where the (small) win from this comes from varies between CPUs, the net effect is a 1 to 2% reduction on a combined interruption entry and exit when the full state save can be avoided. 1: use MOV

...IRQS_OFF jmp int_with_check CFI_ENDPROC @@ -525,7 +533,7 @@ ENTRY(common_interrupt) interrupt do_IRQ /* 0(%rsp): oldrsp-ARGOFFSET */ ret_from_intr: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF decl %gs:pda_irqcount leaveq @@ -552,13 +560,13 @@ retint_swapgs: /* * The iretq could re-enable interrupts: */ - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_IRETQ swapgs jmp restore_args retint_restore_args: - cli + DISABLE_INTERRUPTS(CLBR_NONE) /* * The iretq could re-enable interrupts: */ @@ -566,35 +574,22 @@ retint_restore_args: restore_arg...

...IRQS_OFF jmp int_with_check CFI_ENDPROC @@ -525,7 +533,7 @@ ENTRY(common_interrupt) interrupt do_IRQ /* 0(%rsp): oldrsp-ARGOFFSET */ ret_from_intr: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF decl %gs:pda_irqcount leaveq @@ -552,13 +560,13 @@ retint_swapgs: /* * The iretq could re-enable interrupts: */ - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_IRETQ swapgs jmp restore_args retint_restore_args: - cli + DISABLE_INTERRUPTS(CLBR_NONE) /* * The iretq could re-enable interrupts: */ @@ -566,35 +574,22 @@ retint_restore_args: restore_arg...

...addq $(6*8), %rsp @@ -1576,7 +1578,9 @@ first_nmi: addq $8, (%rsp) /* Fix up RSP */ pushfq /* RFLAGS */ pushq $__KERNEL_CS /* CS */ - pushq $1f /* RIP */ + pushq %rax /* Support Position Independent Code */ + leaq 1f(%rip), %rax /* RIP */ + xchgq %rax, (%rsp) /* Restore RAX, put 1f */ iretq /* continues at repeat_nmi below */ UNWIND_HINT_IRET_REGS 1: diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index a7227dfe1a2b..0c0fc259a4e2 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -208,11 +208,9...

..._KERNEL_CS /* CS */ > > > - pushq $1f /* RIP */ > > > + pushq %rax /* Support Position Independent Code */ > > > + leaq 1f(%rip), %rax /* RIP */ > > > + xchgq %rax, (%rsp) /* Restore RAX, put 1f */ > > > iretq /* continues at repeat_nmi below */ > > > UNWIND_HINT_IRET_REGS > > > 1: > > > > Urgh, xchg with a memop has an implicit LOCK prefix. > this_cpu_xchg uses no lock cmpxchg as a replacement to reduce latency. Great, I will update my implementa...

...t=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_no_ghcb) +#endif #define SYM_DATA_START_PAGE_ALIGNED(name) \ SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) -- 2.28.0

...t=8 + + /* Build pt_regs */ + PUSH_AND_CLEAR_REGS + + /* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + call do_vc_no_ghcb + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_no_ghcb) +#endif #define SYM_DATA_START_PAGE_ALIGNED(name) \ SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index e2777cc264f5..0d560a1218e1 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -11,6 +...

...u struct is allocated (via get_free_pages) and then mapped into the HV FIXMAP area. The host then maps the HV pages and this vcpu data into the guest area in the same place. Then we jump to the hypervisor which changes the gdt idt and cr3 for the guest (as well as the process GS base) and does an iretq into the guest memory. Page faulting: -------------- This is a bit different too. When the guest takes a page fault, we jump back to to the host via switch_to_host, and the host needs to map in the page. Page Hashes ----------- The lguest_guest_info structure holds a bunch of pud, pmd, and p...

...u struct is allocated (via get_free_pages) and then mapped into the HV FIXMAP area. The host then maps the HV pages and this vcpu data into the guest area in the same place. Then we jump to the hypervisor which changes the gdt idt and cr3 for the guest (as well as the process GS base) and does an iretq into the guest memory. Page faulting: -------------- This is a bit different too. When the guest takes a page fault, we jump back to to the host via switch_to_host, and the host needs to map in the page. Page Hashes ----------- The lguest_guest_info structure holds a bunch of pud, pmd, and p...

...estore regs */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + popq %r11 + popq %r10 + popq %r9 + popq %r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + + /* Remove error code and return */ + addq $8, %rsp + + iretq +SYM_FUNC_END(\name) + .endm + + .text + .code64 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 726e264410ff..062ae3ae6930 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -23,6 +23,7 @@ #include <asm/page.h> #includ...

The self_nmi() code cause''s an NMI to be triggered by sending an APIC message to the local processor. However, NMIs are blocked by the VMEXIT, until the next iret or VMENTER. Volume 3 Chapter 27 Section 1 of the Intel SDM states: An NMI causes subsequent NMIs to be blocked, but only after the VM exit completes. As a result, as soon as the VMENTER happens, an immediate VMEXIT happens

...%r8 + popq %rax + popq %rcx + popq %rdx + popq %rsi + popq %rdi + + /* Remove error code and return */ + addq $8, %rsp + + /* + * Make sure we return to __KERNEL_CS - the CS selector on + * the IRET frame might still be from an old BIOS GDT + */ + movq $__KERNEL_CS, 8(%rsp) + + iretq +SYM_FUNC_END(\name) + .endm + + .text + .code64 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 726e264410ff..062ae3ae6930 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -23,6 +23,7 @@ #include <asm/page.h> #includ...

.../* Call C handler */ + movq %rsp, %rdi + movq ORIG_RAX(%rsp), %rsi + movq initial_vc_handler(%rip), %rax + ANNOTATE_RETPOLINE_SAFE + call *%rax + + /* Unwind pt_regs */ + POP_REGS + + /* Remove Error Code */ + addq $8, %rsp + + /* Pure iret required here - don't use INTERRUPT_RETURN */ + iretq +SYM_CODE_END(vc_boot_ghcb) #endif /* Both SMP bootup and ACPI suspend change these variables */ @@ -286,6 +317,9 @@ SYM_CODE_END(start_cpu0) .balign 8 SYM_DATA(initial_code, .quad x86_64_start_kernel) SYM_DATA(initial_gs, .quad INIT_PER_CPU_VAR(fixed_percpu_data)) +#ifdef CONFIG_AMD_MEM_E...

Changes: - patch v2: - Adapt patch to work post KPTI and compiler changes - Redo all performance testing with latest configs and compilers - Simplify mov macro on PIE (MOVABS now) - Reduce GOT footprint - patch v1: - Simplify ftrace implementation. - Use gcc mstack-protector-guard-reg=%gs with PIE when possible. - rfc v3: - Use --emit-relocs instead of -pie to reduce

Changes: - patch v2: - Adapt patch to work post KPTI and compiler changes - Redo all performance testing with latest configs and compilers - Simplify mov macro on PIE (MOVABS now) - Reduce GOT footprint - patch v1: - Simplify ftrace implementation. - Use gcc mstack-protector-guard-reg=%gs with PIE when possible. - rfc v3: - Use --emit-relocs instead of -pie to reduce