search for: config_amd_mem_encrypt

...) /* clear "NMI executing" */ +nmi_return: + UNWIND_HINT_IRET_REGS + /* * iretq reads the "iret" frame and exits the NMI stack in a * single instruction. We are returning to kernel mode, so this @@ -1724,6 +1744,34 @@ nmi_restore: iretq SYM_CODE_END(nmi) +#ifdef CONFIG_AMD_MEM_ENCRYPT +SYM_CODE_START(sev_es_iret_user) + UNWIND_HINT_IRET_REGS offset=8 + /* + * The kernel jumps here directly from + * swapgs_restore_regs_and_return_to_usermode. %rsp points already to + * trampoline stack, but %cr3 is still from kernel. User-regs are live + * except %rdi. Switch to user CR3, res...

...x/types.h> + +#include <asm/stacktrace.h> #include <asm/insn.h> #define GHCB_SEV_INFO 0x001UL @@ -76,4 +78,15 @@ static inline u64 lower_bits(u64 val, unsigned int bits) extern void vc_no_ghcb(void); extern bool vc_boot_ghcb(struct pt_regs *regs); +enum stack_type; + +#ifdef CONFIG_AMD_MEM_ENCRYPT +const char *vc_stack_name(enum stack_type type); +#else /* CONFIG_AMD_MEM_ENCRYPT */ +static inline const char *vc_stack_name(enum stack_type type) +{ + return NULL; +} +#endif /* CONFIG_AMD_MEM_ENCRYPT*/ + #endif diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h...

On Mon, Aug 24, 2020 at 10:54:37AM +0200, Joerg Roedel wrote: > +#ifdef CONFIG_AMD_MEM_ENCRYPT > +static void set_early_idt_handler(gate_desc *idt, int n, void *handler) > +{ > + struct idt_data data; > + gate_desc desc; > + > + init_idt_data(&data, n, handler); > + idt_init_desc(&desc, &data); > + native_write_idt_entry(idt, n, &desc); > +} > +#...

On Mon, Aug 24, 2020 at 10:54:37AM +0200, Joerg Roedel wrote: > +#ifdef CONFIG_AMD_MEM_ENCRYPT > +static void set_early_idt_handler(gate_desc *idt, int n, void *handler) > +{ > + struct idt_data data; > + gate_desc desc; > + > + init_idt_data(&data, n, handler); > + idt_init_desc(&desc, &data); > + native_write_idt_entry(idt, n, &desc); > +} > +#...

...-40,6 +40,7 @@ #include <asm/desc.h> #include <asm/extable.h> #include <asm/trapnr.h> +#include <asm/sev-es.h> /* * Manage page tables very early on. @@ -540,10 +541,29 @@ static struct desc_ptr bringup_idt_descr = { .address = 0, /* Set at runtime */ }; +#ifdef CONFIG_AMD_MEM_ENCRYPT +static void set_bringup_idt_handler(int n, void *handler) +{ + struct idt_data data; + gate_desc desc; + + init_idt_data(&data, n, handler); + idt_init_desc(&desc, &data); + native_write_idt_entry(bringup_idt_table, n, &desc); +} +#endif + /* This runs while still in the direct ma...

...lmode.h index b35030eeec36..96118fb041b8 100644 --- a/arch/x86/include/asm/realmode.h +++ b/arch/x86/include/asm/realmode.h @@ -57,6 +57,9 @@ extern unsigned char real_mode_blob_end[]; extern unsigned long initial_code; extern unsigned long initial_gs; extern unsigned long initial_stack; +#ifdef CONFIG_AMD_MEM_ENCRYPT +extern unsigned long initial_vc_handler; +#endif extern unsigned char real_mode_blob[]; extern unsigned char real_mode_relocs[]; diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index 517920928989..7fdd4facfce7 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/...

...6/entry/entry_64.S index 0e9504fabe52..4c392eb2f063 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1210,6 +1210,10 @@ idtentry async_page_fault do_async_page_fault has_error_code=1 read_cr2=1 idtentry machine_check do_mce has_error_code=0 paranoid=1 #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT +idtentry vmm_communication do_vmm_communication has_error_code=1 paranoid=1 shift_ist=IST_INDEX_VC ist_offset=VC_STACK_OFFSET +#endif + /* * Save all registers in pt_regs, and switch gs if needed. * Use slow, but surefire "are we in kernel?" check. diff --git a/arch/x86/include/asm/...

...se as first parameter */ + leaq _text(%rip), %rdi + call early_idt_setup + popq %rsi + /* Sanitize CPU configuration */ call verify_cpu @@ -363,6 +370,33 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) jmp restore_regs_and_return_to_kernel SYM_CODE_END(early_idt_handler_common) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * VC Exception handler used during very early boot. The + * early_idt_handler_array can't be used because it returns via the + * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. + */ +SYM_CODE_START_NOALIGN(vc_no_ghcb) + UNWIND_HINT_IRET_REGS offset=8 + + /* Build pt...

...e2e6439 100644 --- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -78,17 +78,23 @@ static inline u64 lower_bits(u64 val, unsigned int bits) extern void vc_no_ghcb(void); extern bool vc_boot_ghcb(struct pt_regs *regs); +struct real_mode_header; enum stack_type; #ifdef CONFIG_AMD_MEM_ENCRYPT const char *vc_stack_name(enum stack_type type); void sev_es_nmi_enter(void); void sev_es_nmi_exit(void); +int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); #else /* CONFIG_AMD_MEM_ENCRYPT */ static inline const char *vc_stack_name(enum stack_type type) { return NULL; } +static...

...rcepting those instructions. Ahh, so any instruction that can have an instruction intercept set potentially needs to be able to tolerate a #VC? Those instruction intercepts are under the control of the (untrusted relative to the guest) hypervisor, right? >From the main sev-es series: +#ifdef CONFIG_AMD_MEM_ENCRYPT +idtentry vmm_communication do_vmm_communication has_error_code=1 +#endif Since this is set as non-paranoid, that both limits the instructions that can be used in entry paths *and* limits the future architecture from being able add instructions that a current SEV-ES guest doesn't know a...

...rcepting those instructions. Ahh, so any instruction that can have an instruction intercept set potentially needs to be able to tolerate a #VC? Those instruction intercepts are under the control of the (untrusted relative to the guest) hypervisor, right? >From the main sev-es series: +#ifdef CONFIG_AMD_MEM_ENCRYPT +idtentry vmm_communication do_vmm_communication has_error_code=1 +#endif Since this is set as non-paranoid, that both limits the instructions that can be used in entry paths *and* limits the future architecture from being able add instructions that a current SEV-ES guest doesn't know a...

...de <asm/apic.h> #include <asm/vmware.h> +#include <asm/svm.h> #undef pr_fmt #define pr_fmt(fmt) "vmware: " fmt @@ -476,10 +477,49 @@ static bool __init vmware_legacy_x2apic_available(void) (eax & (1 << VMWARE_CMD_LEGACY_X2APIC)) != 0; } +#ifdef CONFIG_AMD_MEM_ENCRYPT +static void vmware_sev_es_hcall_prepare(struct ghcb *ghcb, + struct pt_regs *regs) +{ + /* Copy VMWARE specific Hypercall parameters to the GHCB */ + ghcb_set_rip(ghcb, regs->ip); + ghcb_set_rbx(ghcb, regs->bx); + ghcb_set_rcx(ghcb, regs->cx); + ghcb_set_rdx(ghcb, regs->dx); + ghcb...

...a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -86,6 +86,7 @@ const char *vc_stack_name(enum stack_type type); void sev_es_nmi_enter(void); void sev_es_nmi_exit(void); int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); +void sev_es_nmi_complete(void); #else /* CONFIG_AMD_MEM_ENCRYPT */ static inline const char *vc_stack_name(enum stack_type type) { @@ -95,6 +96,7 @@ static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } +static inline void sev_es_nmi_complete(void) { } #endif /* CONFIG_AMD_MEM_ENCRYPT*/ #endif diff --git a/arch/x86/i...

...9..a7847a1ef63a 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -85,6 +85,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 99cc78062684..f8295d68b3e1 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -31,6 +31,10 @@ vo...

...e..583678c78e1b 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -84,6 +84,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 84ba57d9d436..bdd20dfd1fd0 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -31,6 +31,10 @@ vo...

...b..7302f184bece 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -83,6 +83,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-y += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 5f083092a86d..f3ca7324be44 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -32,6 +32,10 @@ vo...

...Makefile > +++ b/arch/x86/boot/compressed/Makefile > @@ -84,6 +84,7 @@ ifdef CONFIG_X86_64 > vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o > vmlinux-objs-y += $(obj)/mem_encrypt.o > vmlinux-objs-y += $(obj)/pgtable_64.o > + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o > endif > > vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o > diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c > index 84ba57d9d436..bdd20dfd1fd0 100644 > --- a/arch/x86/boot/compressed/idt_64.c > +++ b/arch/x86/boot/compress...

...Makefile > +++ b/arch/x86/boot/compressed/Makefile > @@ -84,6 +84,7 @@ ifdef CONFIG_X86_64 > vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o > vmlinux-objs-y += $(obj)/mem_encrypt.o > vmlinux-objs-y += $(obj)/pgtable_64.o > + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev-es.o > endif > > vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o > diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c > index 84ba57d9d436..bdd20dfd1fd0 100644 > --- a/arch/x86/boot/compressed/idt_64.c > +++ b/arch/x86/boot/compress...

...--- a/arch/x86/include/asm/sev-es.h +++ b/arch/x86/include/asm/sev-es.h @@ -87,6 +87,7 @@ void sev_es_nmi_enter(void); void sev_es_nmi_exit(void); int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); void sev_es_nmi_complete(void); +int __init sev_es_efi_map_ghcbs(pgd_t *pgd); #else /* CONFIG_AMD_MEM_ENCRYPT */ static inline const char *vc_stack_name(enum stack_type type) { @@ -97,6 +98,10 @@ static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) return 0; } static inline void sev_es_nmi_complete(void) { } +static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) +{ + return 0; +}...

...git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index f8295d68b3e1..44d20c4f47c9 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -45,5 +45,9 @@ void load_stage2_idt(void) set_idt_entry(X86_TRAP_PF, boot_page_fault); +#ifdef CONFIG_AMD_MEM_ENCRYPT + set_idt_entry(X86_TRAP_VC, boot_stage2_vc); +#endif + load_boot_idt(&boot_idt_desc); } diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 8473bf88e64e..bd058aa21e4f 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x8...