+2

arch/x86/Kbuild

reviewed

··· 3 3 # Branch profiling isn't noinstr-safe. Disable it for arch/x86/* 4 4 subdir-ccflags-$(CONFIG_TRACE_BRANCH_PROFILING) += -DDISABLE_BRANCH_PROFILING 5 5 6 6 + obj-y += boot/startup/ 7 7 + 6 8 obj-$(CONFIG_ARCH_HAS_CC_PLATFORM) += coco/ 7 9 8 10 obj-y += entry/

+13

arch/x86/Kconfig

reviewed

··· 487 487 488 488 If in doubt, say Y. 489 489 490 490 + config AMD_SECURE_AVIC 491 491 + bool "AMD Secure AVIC" 492 492 + depends on AMD_MEM_ENCRYPT && X86_X2APIC 493 493 + help 494 494 + Enable this to get AMD Secure AVIC support on guests that have this feature. 495 495 + 496 496 + AMD Secure AVIC provides hardware acceleration for performance sensitive 497 497 + APIC accesses and support for managing guest owned APIC state for SEV-SNP 498 498 + guests. Secure AVIC does not support xAPIC mode. It has functional 499 499 + dependency on x2apic being enabled in the guest. 500 500 + 501 501 + If you don't know what to do here, say N. 502 502 + 490 503 config X86_POSTED_MSI 491 504 bool "Enable MSI and MSI-x delivery by posted interrupts" 492 505 depends on X86_64 && IRQ_REMAP

-1

arch/x86/Makefile

reviewed

··· 275 275 ### 276 276 # Kernel objects 277 277 278 278 - core-y += arch/x86/boot/startup/ 279 278 libs-y += arch/x86/lib/ 280 279 281 280 # drivers-y are linked after core-y

+1 -1

arch/x86/boot/compressed/Makefile

reviewed

··· 73 73 hostprogs := mkpiggy 74 74 HOST_EXTRACFLAGS += -I$(srctree)/tools/include 75 75 76 76 - sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' 76 76 + sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|_sinittext\|__inittext_end\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' 77 77 78 78 quiet_cmd_voffset = VOFFSET $@ 79 79 cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@

+2

arch/x86/boot/compressed/misc.c

reviewed

··· 332 332 } 333 333 334 334 const unsigned long kernel_text_size = VO___start_rodata - VO__text; 335 335 + const unsigned long kernel_inittext_offset = VO__sinittext - VO__text; 336 336 + const unsigned long kernel_inittext_size = VO___inittext_end - VO__sinittext; 335 337 const unsigned long kernel_total_size = VO__end - VO__text; 336 338 337 339 static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4);

+3

arch/x86/boot/compressed/sev-handle-vc.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 3 3 #include "misc.h" 4 4 + #include "error.h" 4 5 #include "sev.h" 5 6 6 7 #include <linux/kernel.h> ··· 15 14 #include <asm/fpu/xcr.h> 16 15 17 16 #define __BOOT_COMPRESSED 17 17 + #undef __init 18 18 + #define __init 18 19 19 20 /* Basic instruction decoding support needed */ 20 21 #include "../../lib/inat.c"

+39 -93

arch/x86/boot/compressed/sev.c

reviewed

··· 32 32 #undef __init 33 33 #define __init 34 34 35 35 - #undef __head 36 36 - #define __head 37 37 - 38 35 #define __BOOT_COMPRESSED 39 36 40 40 - extern struct svsm_ca *boot_svsm_caa; 41 41 - extern u64 boot_svsm_caa_pa; 42 42 - 43 43 - struct svsm_ca *svsm_get_caa(void) 44 44 - { 45 45 - return boot_svsm_caa; 46 46 - } 47 47 - 48 48 - u64 svsm_get_caa_pa(void) 49 49 - { 50 50 - return boot_svsm_caa_pa; 51 51 - } 52 52 - 53 53 - int svsm_perform_call_protocol(struct svsm_call *call); 54 54 - 55 37 u8 snp_vmpl; 38 38 + u16 ghcb_version; 39 39 + 40 40 + u64 boot_svsm_caa_pa; 56 41 57 42 /* Include code for early handlers */ 58 43 #include "../../boot/startup/sev-shared.c" 59 59 - 60 60 - int svsm_perform_call_protocol(struct svsm_call *call) 61 61 - { 62 62 - struct ghcb *ghcb; 63 63 - int ret; 64 64 - 65 65 - if (boot_ghcb) 66 66 - ghcb = boot_ghcb; 67 67 - else 68 68 - ghcb = NULL; 69 69 - 70 70 - do { 71 71 - ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) 72 72 - : svsm_perform_msr_protocol(call); 73 73 - } while (ret == -EAGAIN); 74 74 - 75 75 - return ret; 76 76 - } 77 44 78 45 static bool sev_snp_enabled(void) 79 46 { 80 47 return sev_status & MSR_AMD64_SEV_SNP_ENABLED; 81 48 } 82 49 83 83 - static void __page_state_change(unsigned long paddr, enum psc_op op) 84 84 - { 85 85 - u64 val, msr; 86 86 - 87 87 - /* 88 88 - * If private -> shared then invalidate the page before requesting the 89 89 - * state change in the RMP table. 90 90 - */ 91 91 - if (op == SNP_PAGE_STATE_SHARED) 92 92 - pvalidate_4k_page(paddr, paddr, false); 93 93 - 94 94 - /* Save the current GHCB MSR value */ 95 95 - msr = sev_es_rd_ghcb_msr(); 96 96 - 97 97 - /* Issue VMGEXIT to change the page state in RMP table. */ 98 98 - sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); 99 99 - VMGEXIT(); 100 100 - 101 101 - /* Read the response of the VMGEXIT. */ 102 102 - val = sev_es_rd_ghcb_msr(); 103 103 - if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val)) 104 104 - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 105 105 - 106 106 - /* Restore the GHCB MSR value */ 107 107 - sev_es_wr_ghcb_msr(msr); 108 108 - 109 109 - /* 110 110 - * Now that page state is changed in the RMP table, validate it so that it is 111 111 - * consistent with the RMP entry. 112 112 - */ 113 113 - if (op == SNP_PAGE_STATE_PRIVATE) 114 114 - pvalidate_4k_page(paddr, paddr, true); 115 115 - } 116 116 - 117 50 void snp_set_page_private(unsigned long paddr) 118 51 { 52 52 + struct psc_desc d = { 53 53 + SNP_PAGE_STATE_PRIVATE, 54 54 + (struct svsm_ca *)boot_svsm_caa_pa, 55 55 + boot_svsm_caa_pa 56 56 + }; 57 57 + 119 58 if (!sev_snp_enabled()) 120 59 return; 121 60 122 122 - __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE); 61 61 + __page_state_change(paddr, paddr, &d); 123 62 } 124 63 125 64 void snp_set_page_shared(unsigned long paddr) 126 65 { 66 66 + struct psc_desc d = { 67 67 + SNP_PAGE_STATE_SHARED, 68 68 + (struct svsm_ca *)boot_svsm_caa_pa, 69 69 + boot_svsm_caa_pa 70 70 + }; 71 71 + 127 72 if (!sev_snp_enabled()) 128 73 return; 129 74 130 130 - __page_state_change(paddr, SNP_PAGE_STATE_SHARED); 75 75 + __page_state_change(paddr, paddr, &d); 131 76 } 132 77 133 78 bool early_setup_ghcb(void) ··· 97 152 98 153 void snp_accept_memory(phys_addr_t start, phys_addr_t end) 99 154 { 155 155 + struct psc_desc d = { 156 156 + SNP_PAGE_STATE_PRIVATE, 157 157 + (struct svsm_ca *)boot_svsm_caa_pa, 158 158 + boot_svsm_caa_pa 159 159 + }; 160 160 + 100 161 for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE) 101 101 - __page_state_change(pa, SNP_PAGE_STATE_PRIVATE); 162 162 + __page_state_change(pa, pa, &d); 102 163 } 103 164 104 165 void sev_es_shutdown_ghcb(void) ··· 186 235 MSR_AMD64_SNP_VMSA_REG_PROT | \ 187 236 MSR_AMD64_SNP_RESERVED_BIT13 | \ 188 237 MSR_AMD64_SNP_RESERVED_BIT15 | \ 238 238 + MSR_AMD64_SNP_SECURE_AVIC | \ 189 239 MSR_AMD64_SNP_RESERVED_MASK) 240 240 + 241 241 + #ifdef CONFIG_AMD_SECURE_AVIC 242 242 + #define SNP_FEATURE_SECURE_AVIC MSR_AMD64_SNP_SECURE_AVIC 243 243 + #else 244 244 + #define SNP_FEATURE_SECURE_AVIC 0 245 245 + #endif 190 246 191 247 /* 192 248 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented ··· 201 243 * guest kernel, a corresponding bit should be added to the mask. 202 244 */ 203 245 #define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \ 204 204 - MSR_AMD64_SNP_SECURE_TSC) 246 246 + MSR_AMD64_SNP_SECURE_TSC | \ 247 247 + SNP_FEATURE_SECURE_AVIC) 205 248 206 249 u64 snp_get_unsupported_features(u64 status) 207 250 { ··· 306 347 * running at VMPL0. The CA will be used to communicate with the 307 348 * SVSM and request its services. 308 349 */ 309 309 - svsm_setup_ca(cc_info); 350 350 + svsm_setup_ca(cc_info, rip_rel_ptr(&boot_ghcb_page)); 310 351 311 352 /* 312 353 * Pass run-time kernel a pointer to CC info via boot_params so EFI ··· 349 390 /* Check whether SEV is supported */ 350 391 if (!(eax & BIT(1))) 351 392 return -ENODEV; 393 393 + 394 394 + sev_snp_needs_sfw = !(ebx & BIT(31)); 352 395 353 396 return ebx & 0x3f; 354 397 } ··· 414 453 */ 415 454 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) { 416 455 u64 hv_features; 417 417 - int ret; 418 456 419 457 hv_features = get_hv_features(); 420 458 if (!(hv_features & GHCB_HV_FT_SNP)) 421 459 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 422 460 423 461 /* 424 424 - * Enforce running at VMPL0 or with an SVSM. 425 425 - * 426 426 - * Use RMPADJUST (see the rmpadjust() function for a description of 427 427 - * what the instruction does) to update the VMPL1 permissions of a 428 428 - * page. If the guest is running at VMPL0, this will succeed. If the 429 429 - * guest is running at any other VMPL, this will fail. Linux SNP guests 430 430 - * only ever run at a single VMPL level so permission mask changes of a 431 431 - * lesser-privileged VMPL are a don't-care. 462 462 + * Running at VMPL0 is required unless an SVSM is present and 463 463 + * the hypervisor supports the required SVSM GHCB events. 432 464 */ 433 433 - ret = rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, 1); 434 434 - 435 435 - /* 436 436 - * Running at VMPL0 is not required if an SVSM is present and the hypervisor 437 437 - * supports the required SVSM GHCB events. 438 438 - */ 439 439 - if (ret && 440 440 - !(snp_vmpl && (hv_features & GHCB_HV_FT_SNP_MULTI_VMPL))) 465 465 + if (snp_vmpl && !(hv_features & GHCB_HV_FT_SNP_MULTI_VMPL)) 441 466 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0); 442 467 } 443 468 ··· 497 550 498 551 /* Obtain the address of the calling area to use */ 499 552 boot_rdmsr(MSR_SVSM_CAA, &m); 500 500 - boot_svsm_caa = (void *)m.q; 501 553 boot_svsm_caa_pa = m.q; 502 554 503 555 /*

-13

arch/x86/boot/cpuflags.c

reviewed

··· 106 106 cpuid(0x80000001, &ignored, &ignored, &cpu.flags[6], 107 107 &cpu.flags[1]); 108 108 } 109 109 - 110 110 - if (max_amd_level >= 0x8000001f) { 111 111 - u32 ebx; 112 112 - 113 113 - /* 114 114 - * The X86_FEATURE_COHERENCY_SFW_NO feature bit is in 115 115 - * the virtualization flags entry (word 8) and set by 116 116 - * scattered.c, so the bit needs to be explicitly set. 117 117 - */ 118 118 - cpuid(0x8000001f, &ignored, &ebx, &ignored, &ignored); 119 119 - if (ebx & BIT(31)) 120 120 - set_bit(X86_FEATURE_COHERENCY_SFW_NO, cpu.flags); 121 121 - } 122 109 } 123 110 }

+22

arch/x86/boot/startup/Makefile

reviewed

··· 4 4 KBUILD_CFLAGS += -D__DISABLE_EXPORTS -mcmodel=small -fPIC \ 5 5 -Os -DDISABLE_BRANCH_PROFILING \ 6 6 $(DISABLE_STACKLEAK_PLUGIN) \ 7 7 + $(DISABLE_LATENT_ENTROPY_PLUGIN) \ 7 8 -fno-stack-protector -D__NO_FORTIFY \ 8 9 -fno-jump-tables \ 9 10 -include $(srctree)/include/linux/hidden.h ··· 20 19 21 20 obj-$(CONFIG_X86_64) += gdt_idt.o map_kernel.o 22 21 obj-$(CONFIG_AMD_MEM_ENCRYPT) += sme.o sev-startup.o 22 22 + pi-objs := $(patsubst %.o,$(obj)/%.o,$(obj-y)) 23 23 24 24 lib-$(CONFIG_X86_64) += la57toggle.o 25 25 lib-$(CONFIG_EFI_MIXED) += efi-mixed.o ··· 30 28 # to be linked into the decompressor or the EFI stub but not vmlinux 31 29 # 32 30 $(patsubst %.o,$(obj)/%.o,$(lib-y)): OBJECT_FILES_NON_STANDARD := y 31 31 + 32 32 + # 33 33 + # Invoke objtool for each object individually to check for absolute 34 34 + # relocations, even if other objtool actions are being deferred. 35 35 + # 36 36 + $(pi-objs): objtool-enabled = 1 37 37 + $(pi-objs): objtool-args = $(if $(delay-objtool),,$(objtool-args-y)) --noabs 38 38 + 39 39 + # 40 40 + # Confine the startup code by prefixing all symbols with __pi_ (for position 41 41 + # independent). This ensures that startup code can only call other startup 42 42 + # code, or code that has explicitly been made accessible to it via a symbol 43 43 + # alias. 44 44 + # 45 45 + $(obj)/%.pi.o: OBJCOPYFLAGS := --prefix-symbols=__pi_ 46 46 + $(obj)/%.pi.o: $(obj)/%.o FORCE 47 47 + $(call if_changed,objcopy) 48 48 + 49 49 + targets += $(obj-y) 50 50 + obj-y := $(patsubst %.o,%.pi.o,$(obj-y))

+14

arch/x86/boot/startup/exports.h

reviewed

··· 1 1 + 2 2 + /* 3 3 + * The symbols below are functions that are implemented by the startup code, 4 4 + * but called at runtime by the SEV code residing in the core kernel. 5 5 + */ 6 6 + PROVIDE(early_set_pages_state = __pi_early_set_pages_state); 7 7 + PROVIDE(early_snp_set_memory_private = __pi_early_snp_set_memory_private); 8 8 + PROVIDE(early_snp_set_memory_shared = __pi_early_snp_set_memory_shared); 9 9 + PROVIDE(get_hv_features = __pi_get_hv_features); 10 10 + PROVIDE(sev_es_terminate = __pi_sev_es_terminate); 11 11 + PROVIDE(snp_cpuid = __pi_snp_cpuid); 12 12 + PROVIDE(snp_cpuid_get_table = __pi_snp_cpuid_get_table); 13 13 + PROVIDE(svsm_issue_call = __pi_svsm_issue_call); 14 14 + PROVIDE(svsm_process_result_codes = __pi_svsm_process_result_codes);

+2 -2

arch/x86/boot/startup/gdt_idt.c

reviewed

··· 24 24 static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; 25 25 26 26 /* This may run while still in the direct mapping */ 27 27 - void __head startup_64_load_idt(void *vc_handler) 27 27 + void startup_64_load_idt(void *vc_handler) 28 28 { 29 29 struct desc_ptr desc = { 30 30 .address = (unsigned long)rip_rel_ptr(bringup_idt_table), ··· 46 46 /* 47 47 * Setup boot CPU state needed before kernel switches to virtual addresses. 48 48 */ 49 49 - void __head startup_64_setup_gdt_idt(void) 49 49 + void __init startup_64_setup_gdt_idt(void) 50 50 { 51 51 struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page); 52 52 void *handler = NULL;

+2 -2

arch/x86/boot/startup/map_kernel.c

reviewed

··· 30 30 return true; 31 31 } 32 32 33 33 - static unsigned long __head sme_postprocess_startup(struct boot_params *bp, 33 33 + static unsigned long __init sme_postprocess_startup(struct boot_params *bp, 34 34 pmdval_t *pmd, 35 35 unsigned long p2v_offset) 36 36 { ··· 84 84 * the 1:1 mapping of memory. Kernel virtual addresses can be determined by 85 85 * subtracting p2v_offset from the RIP-relative address. 86 86 */ 87 87 - unsigned long __head __startup_64(unsigned long p2v_offset, 87 87 + unsigned long __init __startup_64(unsigned long p2v_offset, 88 88 struct boot_params *bp) 89 89 { 90 90 pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);

+97 -230

arch/x86/boot/startup/sev-shared.c

reviewed

··· 12 12 #include <asm/setup_data.h> 13 13 14 14 #ifndef __BOOT_COMPRESSED 15 15 - #define error(v) pr_err(v) 16 15 #define has_cpuflag(f) boot_cpu_has(f) 17 16 #else 18 17 #undef WARN 19 18 #define WARN(condition, format...) (!!(condition)) 20 20 - #undef vc_forward_exception 21 21 - #define vc_forward_exception(c) panic("SNP: Hypervisor requested exception\n") 22 19 #endif 23 23 - 24 24 - /* 25 25 - * SVSM related information: 26 26 - * During boot, the page tables are set up as identity mapped and later 27 27 - * changed to use kernel virtual addresses. Maintain separate virtual and 28 28 - * physical addresses for the CAA to allow SVSM functions to be used during 29 29 - * early boot, both with identity mapped virtual addresses and proper kernel 30 30 - * virtual addresses. 31 31 - */ 32 32 - struct svsm_ca *boot_svsm_caa __ro_after_init; 33 33 - u64 boot_svsm_caa_pa __ro_after_init; 34 34 - 35 35 - /* 36 36 - * Since feature negotiation related variables are set early in the boot 37 37 - * process they must reside in the .data section so as not to be zeroed 38 38 - * out when the .bss section is later cleared. 39 39 - * 40 40 - * GHCB protocol version negotiated with the hypervisor. 41 41 - */ 42 42 - static u16 ghcb_version __ro_after_init; 43 20 44 21 /* Copy of the SNP firmware's CPUID page. */ 45 22 static struct snp_cpuid_table cpuid_table_copy __ro_after_init; ··· 31 54 static u32 cpuid_hyp_range_max __ro_after_init; 32 55 static u32 cpuid_ext_range_max __ro_after_init; 33 56 34 34 - bool __init sev_es_check_cpu_features(void) 35 35 - { 36 36 - if (!has_cpuflag(X86_FEATURE_RDRAND)) { 37 37 - error("RDRAND instruction not supported - no trusted source of randomness available\n"); 38 38 - return false; 39 39 - } 57 57 + bool sev_snp_needs_sfw; 40 58 41 41 - return true; 42 42 - } 43 43 - 44 44 - void __head __noreturn 59 59 + void __noreturn 45 60 sev_es_terminate(unsigned int set, unsigned int reason) 46 61 { 47 62 u64 val = GHCB_MSR_TERM_REQ; ··· 52 83 /* 53 84 * The hypervisor features are available from GHCB version 2 onward. 54 85 */ 55 55 - u64 get_hv_features(void) 86 86 + u64 __init get_hv_features(void) 56 87 { 57 88 u64 val; 58 89 ··· 69 100 return GHCB_MSR_HV_FT_RESP_VAL(val); 70 101 } 71 102 72 72 - void snp_register_ghcb_early(unsigned long paddr) 73 73 - { 74 74 - unsigned long pfn = paddr >> PAGE_SHIFT; 75 75 - u64 val; 76 76 - 77 77 - sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 78 78 - VMGEXIT(); 79 79 - 80 80 - val = sev_es_rd_ghcb_msr(); 81 81 - 82 82 - /* If the response GPA is not ours then abort the guest */ 83 83 - if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 84 84 - (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 85 85 - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 86 86 - } 87 87 - 88 88 - bool sev_es_negotiate_protocol(void) 89 89 - { 90 90 - u64 val; 91 91 - 92 92 - /* Do the GHCB protocol version negotiation */ 93 93 - sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 94 94 - VMGEXIT(); 95 95 - val = sev_es_rd_ghcb_msr(); 96 96 - 97 97 - if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 98 98 - return false; 99 99 - 100 100 - if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 101 101 - GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 102 102 - return false; 103 103 - 104 104 - ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 105 105 - 106 106 - return true; 107 107 - } 108 108 - 109 109 - static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 110 110 - { 111 111 - u32 ret; 112 112 - 113 113 - ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 114 114 - if (!ret) 115 115 - return ES_OK; 116 116 - 117 117 - if (ret == 1) { 118 118 - u64 info = ghcb->save.sw_exit_info_2; 119 119 - unsigned long v = info & SVM_EVTINJ_VEC_MASK; 120 120 - 121 121 - /* Check if exception information from hypervisor is sane. */ 122 122 - if ((info & SVM_EVTINJ_VALID) && 123 123 - ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 124 124 - ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 125 125 - ctxt->fi.vector = v; 126 126 - 127 127 - if (info & SVM_EVTINJ_VALID_ERR) 128 128 - ctxt->fi.error_code = info >> 32; 129 129 - 130 130 - return ES_EXCEPTION; 131 131 - } 132 132 - } 133 133 - 134 134 - return ES_VMM_ERROR; 135 135 - } 136 136 - 137 137 - static inline int svsm_process_result_codes(struct svsm_call *call) 103 103 + int svsm_process_result_codes(struct svsm_call *call) 138 104 { 139 105 switch (call->rax_out) { 140 106 case SVSM_SUCCESS: ··· 97 193 * - RAX specifies the SVSM protocol/callid as input and the return code 98 194 * as output. 99 195 */ 100 100 - static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending) 196 196 + void svsm_issue_call(struct svsm_call *call, u8 *pending) 101 197 { 102 198 register unsigned long rax asm("rax") = call->rax; 103 199 register unsigned long rcx asm("rcx") = call->rcx; ··· 120 216 call->r9_out = r9; 121 217 } 122 218 123 123 - static int svsm_perform_msr_protocol(struct svsm_call *call) 219 219 + int svsm_perform_msr_protocol(struct svsm_call *call) 124 220 { 125 221 u8 pending = 0; 126 222 u64 val, resp; ··· 149 245 return -EINVAL; 150 246 151 247 return svsm_process_result_codes(call); 152 152 - } 153 153 - 154 154 - static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) 155 155 - { 156 156 - struct es_em_ctxt ctxt; 157 157 - u8 pending = 0; 158 158 - 159 159 - vc_ghcb_invalidate(ghcb); 160 160 - 161 161 - /* 162 162 - * Fill in protocol and format specifiers. This can be called very early 163 163 - * in the boot, so use rip-relative references as needed. 164 164 - */ 165 165 - ghcb->protocol_version = ghcb_version; 166 166 - ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 167 167 - 168 168 - ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL); 169 169 - ghcb_set_sw_exit_info_1(ghcb, 0); 170 170 - ghcb_set_sw_exit_info_2(ghcb, 0); 171 171 - 172 172 - sev_es_wr_ghcb_msr(__pa(ghcb)); 173 173 - 174 174 - svsm_issue_call(call, &pending); 175 175 - 176 176 - if (pending) 177 177 - return -EINVAL; 178 178 - 179 179 - switch (verify_exception_info(ghcb, &ctxt)) { 180 180 - case ES_OK: 181 181 - break; 182 182 - case ES_EXCEPTION: 183 183 - vc_forward_exception(&ctxt); 184 184 - fallthrough; 185 185 - default: 186 186 - return -EINVAL; 187 187 - } 188 188 - 189 189 - return svsm_process_result_codes(call); 190 190 - } 191 191 - 192 192 - enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 193 193 - struct es_em_ctxt *ctxt, 194 194 - u64 exit_code, u64 exit_info_1, 195 195 - u64 exit_info_2) 196 196 - { 197 197 - /* Fill in protocol and format specifiers */ 198 198 - ghcb->protocol_version = ghcb_version; 199 199 - ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 200 200 - 201 201 - ghcb_set_sw_exit_code(ghcb, exit_code); 202 202 - ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 203 203 - ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 204 204 - 205 205 - sev_es_wr_ghcb_msr(__pa(ghcb)); 206 206 - VMGEXIT(); 207 207 - 208 208 - return verify_exception_info(ghcb, ctxt); 209 248 } 210 249 211 250 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg) ··· 189 342 return ret; 190 343 } 191 344 192 192 - static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 193 193 - { 194 194 - u32 cr4 = native_read_cr4(); 195 195 - int ret; 196 345 197 197 - ghcb_set_rax(ghcb, leaf->fn); 198 198 - ghcb_set_rcx(ghcb, leaf->subfn); 199 199 - 200 200 - if (cr4 & X86_CR4_OSXSAVE) 201 201 - /* Safe to read xcr0 */ 202 202 - ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 203 203 - else 204 204 - /* xgetbv will cause #UD - use reset value for xcr0 */ 205 205 - ghcb_set_xcr0(ghcb, 1); 206 206 - 207 207 - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 208 208 - if (ret != ES_OK) 209 209 - return ret; 210 210 - 211 211 - if (!(ghcb_rax_is_valid(ghcb) && 212 212 - ghcb_rbx_is_valid(ghcb) && 213 213 - ghcb_rcx_is_valid(ghcb) && 214 214 - ghcb_rdx_is_valid(ghcb))) 215 215 - return ES_VMM_ERROR; 216 216 - 217 217 - leaf->eax = ghcb->save.rax; 218 218 - leaf->ebx = ghcb->save.rbx; 219 219 - leaf->ecx = ghcb->save.rcx; 220 220 - leaf->edx = ghcb->save.rdx; 221 221 - 222 222 - return ES_OK; 223 223 - } 224 224 - 225 225 - static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 226 226 - { 227 227 - return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf) 228 228 - : __sev_cpuid_hv_msr(leaf); 229 229 - } 230 346 231 347 /* 232 348 * This may be called early while still running on the initial identity ··· 222 412 * 223 413 * Return: XSAVE area size on success, 0 otherwise. 224 414 */ 225 225 - static u32 __head snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 415 415 + static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 226 416 { 227 417 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 228 418 u64 xfeatures_found = 0; ··· 258 448 return xsave_size; 259 449 } 260 450 261 261 - static bool __head 451 451 + static bool 262 452 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) 263 453 { 264 454 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); ··· 294 484 return false; 295 485 } 296 486 297 297 - static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 487 487 + static void snp_cpuid_hv_msr(void *ctx, struct cpuid_leaf *leaf) 298 488 { 299 299 - if (sev_cpuid_hv(ghcb, ctxt, leaf)) 489 489 + if (__sev_cpuid_hv_msr(leaf)) 300 490 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 301 491 } 302 492 303 303 - static int __head 304 304 - snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 305 305 - struct cpuid_leaf *leaf) 493 493 + static int 494 494 + snp_cpuid_postprocess(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf), 495 495 + void *ctx, struct cpuid_leaf *leaf) 306 496 { 307 497 struct cpuid_leaf leaf_hv = *leaf; 308 498 309 499 switch (leaf->fn) { 310 500 case 0x1: 311 311 - snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 501 501 + cpuid_fn(ctx, &leaf_hv); 312 502 313 503 /* initial APIC ID */ 314 504 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0)); ··· 327 517 break; 328 518 case 0xB: 329 519 leaf_hv.subfn = 0; 330 330 - snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 520 520 + cpuid_fn(ctx, &leaf_hv); 331 521 332 522 /* extended APIC ID */ 333 523 leaf->edx = leaf_hv.edx; ··· 375 565 } 376 566 break; 377 567 case 0x8000001E: 378 378 - snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 568 568 + cpuid_fn(ctx, &leaf_hv); 379 569 380 570 /* extended APIC ID */ 381 571 leaf->eax = leaf_hv.eax; ··· 396 586 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value 397 587 * should be treated as fatal by caller. 398 588 */ 399 399 - int __head 400 400 - snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 589 589 + int snp_cpuid(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf), 590 590 + void *ctx, struct cpuid_leaf *leaf) 401 591 { 402 592 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 403 593 ··· 431 621 return 0; 432 622 } 433 623 434 434 - return snp_cpuid_postprocess(ghcb, ctxt, leaf); 624 624 + return snp_cpuid_postprocess(cpuid_fn, ctx, leaf); 435 625 } 436 626 437 627 /* ··· 439 629 * page yet, so it only supports the MSR based communication with the 440 630 * hypervisor and only the CPUID exit-code. 441 631 */ 442 442 - void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 632 632 + void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 443 633 { 444 634 unsigned int subfn = lower_bits(regs->cx, 32); 445 635 unsigned int fn = lower_bits(regs->ax, 32); ··· 458 648 leaf.fn = fn; 459 649 leaf.subfn = subfn; 460 650 461 461 - ret = snp_cpuid(NULL, NULL, &leaf); 651 651 + /* 652 652 + * If SNP is active, then snp_cpuid() uses the CPUID table to obtain the 653 653 + * CPUID values (with possible HV interaction during post-processing of 654 654 + * the values). But if SNP is not active (no CPUID table present), then 655 655 + * snp_cpuid() returns -EOPNOTSUPP so that an SEV-ES guest can call the 656 656 + * HV to obtain the CPUID information. 657 657 + */ 658 658 + ret = snp_cpuid(snp_cpuid_hv_msr, NULL, &leaf); 462 659 if (!ret) 463 660 goto cpuid_done; 464 661 465 662 if (ret != -EOPNOTSUPP) 466 663 goto fail; 467 664 665 665 + /* 666 666 + * This is reached by a SEV-ES guest and needs to invoke the HV for 667 667 + * the CPUID data. 668 668 + */ 468 669 if (__sev_cpuid_hv_msr(&leaf)) 469 670 goto fail; 470 671 ··· 526 705 * Search for a Confidential Computing blob passed in as a setup_data entry 527 706 * via the Linux Boot Protocol. 528 707 */ 529 529 - static __head 708 708 + static __init 530 709 struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) 531 710 { 532 711 struct cc_setup_data *sd = NULL; ··· 554 733 * mapping needs to be updated in sync with all the changes to virtual memory 555 734 * layout and related mapping facilities throughout the boot process. 556 735 */ 557 557 - static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 736 736 + static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 558 737 { 559 738 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; 560 739 int i; ··· 582 761 } 583 762 } 584 763 585 585 - static void __head svsm_pval_4k_page(unsigned long paddr, bool validate) 764 764 + static int svsm_call_msr_protocol(struct svsm_call *call) 765 765 + { 766 766 + int ret; 767 767 + 768 768 + do { 769 769 + ret = svsm_perform_msr_protocol(call); 770 770 + } while (ret == -EAGAIN); 771 771 + 772 772 + return ret; 773 773 + } 774 774 + 775 775 + static void svsm_pval_4k_page(unsigned long paddr, bool validate, 776 776 + struct svsm_ca *caa, u64 caa_pa) 586 777 { 587 778 struct svsm_pvalidate_call *pc; 588 779 struct svsm_call call = {}; 589 780 unsigned long flags; 590 781 u64 pc_pa; 591 591 - int ret; 592 782 593 783 /* 594 784 * This can be called very early in the boot, use native functions in ··· 607 775 */ 608 776 flags = native_local_irq_save(); 609 777 610 610 - call.caa = svsm_get_caa(); 778 778 + call.caa = caa; 611 779 612 780 pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer; 613 613 - pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); 781 781 + pc_pa = caa_pa + offsetof(struct svsm_ca, svsm_buffer); 614 782 615 783 pc->num_entries = 1; 616 784 pc->cur_index = 0; ··· 624 792 call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE); 625 793 call.rcx = pc_pa; 626 794 627 627 - ret = svsm_perform_call_protocol(&call); 628 628 - if (ret) 795 795 + /* 796 796 + * Use the MSR protocol exclusively, so that this code is usable in 797 797 + * startup code where VA/PA translations of the GHCB page's address may 798 798 + * be problematic. 799 799 + */ 800 800 + if (svsm_call_msr_protocol(&call)) 629 801 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 630 802 631 803 native_local_irq_restore(flags); 632 804 } 633 805 634 634 - static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, 635 635 - bool validate) 806 806 + static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, 807 807 + bool validate, struct svsm_ca *caa, u64 caa_pa) 636 808 { 637 809 int ret; 638 810 639 811 if (snp_vmpl) { 640 640 - svsm_pval_4k_page(paddr, validate); 812 812 + svsm_pval_4k_page(paddr, validate, caa, caa_pa); 641 813 } else { 642 814 ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 643 815 if (ret) ··· 652 816 * If validating memory (making it private) and affected by the 653 817 * cache-coherency vulnerability, perform the cache eviction mitigation. 654 818 */ 655 655 - if (validate && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO)) 819 819 + if (validate && sev_snp_needs_sfw) 656 820 sev_evict_cache((void *)vaddr, 1); 821 821 + } 822 822 + 823 823 + static void __page_state_change(unsigned long vaddr, unsigned long paddr, 824 824 + const struct psc_desc *desc) 825 825 + { 826 826 + u64 val, msr; 827 827 + 828 828 + /* 829 829 + * If private -> shared then invalidate the page before requesting the 830 830 + * state change in the RMP table. 831 831 + */ 832 832 + if (desc->op == SNP_PAGE_STATE_SHARED) 833 833 + pvalidate_4k_page(vaddr, paddr, false, desc->ca, desc->caa_pa); 834 834 + 835 835 + /* Save the current GHCB MSR value */ 836 836 + msr = sev_es_rd_ghcb_msr(); 837 837 + 838 838 + /* Issue VMGEXIT to change the page state in RMP table. */ 839 839 + sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, desc->op)); 840 840 + VMGEXIT(); 841 841 + 842 842 + /* Read the response of the VMGEXIT. */ 843 843 + val = sev_es_rd_ghcb_msr(); 844 844 + if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val)) 845 845 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 846 846 + 847 847 + /* Restore the GHCB MSR value */ 848 848 + sev_es_wr_ghcb_msr(msr); 849 849 + 850 850 + /* 851 851 + * Now that page state is changed in the RMP table, validate it so that it is 852 852 + * consistent with the RMP entry. 853 853 + */ 854 854 + if (desc->op == SNP_PAGE_STATE_PRIVATE) 855 855 + pvalidate_4k_page(vaddr, paddr, true, desc->ca, desc->caa_pa); 657 856 } 658 857 659 858 /* 660 859 * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM 661 860 * services needed when not running in VMPL0. 662 861 */ 663 663 - static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) 862 862 + static bool __init svsm_setup_ca(const struct cc_blob_sev_info *cc_info, 863 863 + void *page) 664 864 { 665 865 struct snp_secrets_page *secrets_page; 666 866 struct snp_cpuid_table *cpuid_table; ··· 719 847 * routine is running identity mapped when called, both by the decompressor 720 848 * code and the early kernel code. 721 849 */ 722 722 - if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1)) 850 850 + if (!rmpadjust((unsigned long)page, RMP_PG_SIZE_4K, 1)) 723 851 return false; 724 852 725 853 /* ··· 747 875 if (caa & (PAGE_SIZE - 1)) 748 876 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CAA); 749 877 750 750 - /* 751 751 - * The CA is identity mapped when this routine is called, both by the 752 752 - * decompressor code and the early kernel code. 753 753 - */ 754 754 - boot_svsm_caa = (struct svsm_ca *)caa; 755 878 boot_svsm_caa_pa = caa; 756 879 757 880 /* Advertise the SVSM presence via CPUID. */

+31 -179

arch/x86/boot/startup/sev-startup.c

reviewed

··· 41 41 #include <asm/cpuid/api.h> 42 42 #include <asm/cmdline.h> 43 43 44 44 - /* For early boot hypervisor communication in SEV-ES enabled guests */ 45 45 - struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); 46 46 - 47 47 - /* 48 48 - * Needs to be in the .data section because we need it NULL before bss is 49 49 - * cleared 50 50 - */ 51 51 - struct ghcb *boot_ghcb __section(".data"); 52 52 - 53 53 - /* Bitmap of SEV features supported by the hypervisor */ 54 54 - u64 sev_hv_features __ro_after_init; 55 55 - 56 56 - /* Secrets page physical address from the CC blob */ 57 57 - u64 sev_secrets_pa __ro_after_init; 58 58 - 59 59 - /* For early boot SVSM communication */ 60 60 - struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); 61 61 - 62 62 - DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); 63 63 - DEFINE_PER_CPU(u64, svsm_caa_pa); 64 64 - 65 65 - /* 66 66 - * Nothing shall interrupt this code path while holding the per-CPU 67 67 - * GHCB. The backup GHCB is only for NMIs interrupting this path. 68 68 - * 69 69 - * Callers must disable local interrupts around it. 70 70 - */ 71 71 - noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) 72 72 - { 73 73 - struct sev_es_runtime_data *data; 74 74 - struct ghcb *ghcb; 75 75 - 76 76 - WARN_ON(!irqs_disabled()); 77 77 - 78 78 - data = this_cpu_read(runtime_data); 79 79 - ghcb = &data->ghcb_page; 80 80 - 81 81 - if (unlikely(data->ghcb_active)) { 82 82 - /* GHCB is already in use - save its contents */ 83 83 - 84 84 - if (unlikely(data->backup_ghcb_active)) { 85 85 - /* 86 86 - * Backup-GHCB is also already in use. There is no way 87 87 - * to continue here so just kill the machine. To make 88 88 - * panic() work, mark GHCBs inactive so that messages 89 89 - * can be printed out. 90 90 - */ 91 91 - data->ghcb_active = false; 92 92 - data->backup_ghcb_active = false; 93 93 - 94 94 - instrumentation_begin(); 95 95 - panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); 96 96 - instrumentation_end(); 97 97 - } 98 98 - 99 99 - /* Mark backup_ghcb active before writing to it */ 100 100 - data->backup_ghcb_active = true; 101 101 - 102 102 - state->ghcb = &data->backup_ghcb; 103 103 - 104 104 - /* Backup GHCB content */ 105 105 - *state->ghcb = *ghcb; 106 106 - } else { 107 107 - state->ghcb = NULL; 108 108 - data->ghcb_active = true; 109 109 - } 110 110 - 111 111 - return ghcb; 112 112 - } 113 113 - 114 44 /* Include code shared with pre-decompression boot stage */ 115 45 #include "sev-shared.c" 116 46 117 117 - noinstr void __sev_put_ghcb(struct ghcb_state *state) 118 118 - { 119 119 - struct sev_es_runtime_data *data; 120 120 - struct ghcb *ghcb; 121 121 - 122 122 - WARN_ON(!irqs_disabled()); 123 123 - 124 124 - data = this_cpu_read(runtime_data); 125 125 - ghcb = &data->ghcb_page; 126 126 - 127 127 - if (state->ghcb) { 128 128 - /* Restore GHCB from Backup */ 129 129 - *ghcb = *state->ghcb; 130 130 - data->backup_ghcb_active = false; 131 131 - state->ghcb = NULL; 132 132 - } else { 133 133 - /* 134 134 - * Invalidate the GHCB so a VMGEXIT instruction issued 135 135 - * from userspace won't appear to be valid. 136 136 - */ 137 137 - vc_ghcb_invalidate(ghcb); 138 138 - data->ghcb_active = false; 139 139 - } 140 140 - } 141 141 - 142 142 - int svsm_perform_call_protocol(struct svsm_call *call) 143 143 - { 144 144 - struct ghcb_state state; 145 145 - unsigned long flags; 146 146 - struct ghcb *ghcb; 147 147 - int ret; 148 148 - 149 149 - /* 150 150 - * This can be called very early in the boot, use native functions in 151 151 - * order to avoid paravirt issues. 152 152 - */ 153 153 - flags = native_local_irq_save(); 154 154 - 155 155 - if (sev_cfg.ghcbs_initialized) 156 156 - ghcb = __sev_get_ghcb(&state); 157 157 - else if (boot_ghcb) 158 158 - ghcb = boot_ghcb; 159 159 - else 160 160 - ghcb = NULL; 161 161 - 162 162 - do { 163 163 - ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) 164 164 - : svsm_perform_msr_protocol(call); 165 165 - } while (ret == -EAGAIN); 166 166 - 167 167 - if (sev_cfg.ghcbs_initialized) 168 168 - __sev_put_ghcb(&state); 169 169 - 170 170 - native_local_irq_restore(flags); 171 171 - 172 172 - return ret; 173 173 - } 174 174 - 175 175 - void __head 47 47 + void 176 48 early_set_pages_state(unsigned long vaddr, unsigned long paddr, 177 177 - unsigned long npages, enum psc_op op) 49 49 + unsigned long npages, const struct psc_desc *desc) 178 50 { 179 51 unsigned long paddr_end; 180 180 - u64 val; 181 52 182 53 vaddr = vaddr & PAGE_MASK; 183 54 ··· 56 185 paddr_end = paddr + (npages << PAGE_SHIFT); 57 186 58 187 while (paddr < paddr_end) { 59 59 - /* Page validation must be rescinded before changing to shared */ 60 60 - if (op == SNP_PAGE_STATE_SHARED) 61 61 - pvalidate_4k_page(vaddr, paddr, false); 62 62 - 63 63 - /* 64 64 - * Use the MSR protocol because this function can be called before 65 65 - * the GHCB is established. 66 66 - */ 67 67 - sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); 68 68 - VMGEXIT(); 69 69 - 70 70 - val = sev_es_rd_ghcb_msr(); 71 71 - 72 72 - if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) 73 73 - goto e_term; 74 74 - 75 75 - if (GHCB_MSR_PSC_RESP_VAL(val)) 76 76 - goto e_term; 77 77 - 78 78 - /* Page validation must be performed after changing to private */ 79 79 - if (op == SNP_PAGE_STATE_PRIVATE) 80 80 - pvalidate_4k_page(vaddr, paddr, true); 188 188 + __page_state_change(vaddr, paddr, desc); 81 189 82 190 vaddr += PAGE_SIZE; 83 191 paddr += PAGE_SIZE; 84 192 } 85 85 - 86 86 - return; 87 87 - 88 88 - e_term: 89 89 - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 90 193 } 91 194 92 92 - void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 195 195 + void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 93 196 unsigned long npages) 94 197 { 198 198 + struct psc_desc d = { 199 199 + SNP_PAGE_STATE_PRIVATE, 200 200 + rip_rel_ptr(&boot_svsm_ca_page), 201 201 + boot_svsm_caa_pa 202 202 + }; 203 203 + 95 204 /* 96 205 * This can be invoked in early boot while running identity mapped, so 97 206 * use an open coded check for SNP instead of using cc_platform_has(). ··· 85 234 * Ask the hypervisor to mark the memory pages as private in the RMP 86 235 * table. 87 236 */ 88 88 - early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE); 237 237 + early_set_pages_state(vaddr, paddr, npages, &d); 89 238 } 90 239 91 91 - void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, 240 240 + void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, 92 241 unsigned long npages) 93 242 { 243 243 + struct psc_desc d = { 244 244 + SNP_PAGE_STATE_SHARED, 245 245 + rip_rel_ptr(&boot_svsm_ca_page), 246 246 + boot_svsm_caa_pa 247 247 + }; 248 248 + 94 249 /* 95 250 * This can be invoked in early boot while running identity mapped, so 96 251 * use an open coded check for SNP instead of using cc_platform_has(). ··· 107 250 return; 108 251 109 252 /* Ask hypervisor to mark the memory pages shared in the RMP table. */ 110 110 - early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); 253 253 + early_set_pages_state(vaddr, paddr, npages, &d); 111 254 } 112 255 113 256 /* ··· 123 266 * 124 267 * Scan for the blob in that order. 125 268 */ 126 126 - static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) 269 269 + static struct cc_blob_sev_info *__init find_cc_blob(struct boot_params *bp) 127 270 { 128 271 struct cc_blob_sev_info *cc_info; 129 272 ··· 144 287 145 288 found_cc_info: 146 289 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) 147 147 - snp_abort(); 290 290 + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 148 291 149 292 return cc_info; 150 293 } 151 294 152 152 - static __head void svsm_setup(struct cc_blob_sev_info *cc_info) 295 295 + static void __init svsm_setup(struct cc_blob_sev_info *cc_info) 153 296 { 297 297 + struct snp_secrets_page *secrets = (void *)cc_info->secrets_phys; 154 298 struct svsm_call call = {}; 155 155 - int ret; 156 299 u64 pa; 157 300 158 301 /* ··· 160 303 * running at VMPL0. The CA will be used to communicate with the 161 304 * SVSM to perform the SVSM services. 162 305 */ 163 163 - if (!svsm_setup_ca(cc_info)) 306 306 + if (!svsm_setup_ca(cc_info, rip_rel_ptr(&boot_svsm_ca_page))) 164 307 return; 165 308 166 309 /* ··· 172 315 pa = (u64)rip_rel_ptr(&boot_svsm_ca_page); 173 316 174 317 /* 175 175 - * Switch over to the boot SVSM CA while the current CA is still 176 176 - * addressable. There is no GHCB at this point so use the MSR protocol. 318 318 + * Switch over to the boot SVSM CA while the current CA is still 1:1 319 319 + * mapped and thus addressable with VA == PA. There is no GHCB at this 320 320 + * point so use the MSR protocol. 177 321 * 178 322 * SVSM_CORE_REMAP_CA call: 179 323 * RAX = 0 (Protocol=0, CallID=0) 180 324 * RCX = New CA GPA 181 325 */ 182 182 - call.caa = svsm_get_caa(); 326 326 + call.caa = (struct svsm_ca *)secrets->svsm_caa; 183 327 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 184 328 call.rcx = pa; 185 185 - ret = svsm_perform_call_protocol(&call); 186 186 - if (ret) 329 329 + 330 330 + if (svsm_call_msr_protocol(&call)) 187 331 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL); 188 332 189 189 - boot_svsm_caa = (struct svsm_ca *)pa; 190 333 boot_svsm_caa_pa = pa; 191 334 } 192 335 193 193 - bool __head snp_init(struct boot_params *bp) 336 336 + bool __init snp_init(struct boot_params *bp) 194 337 { 195 338 struct cc_blob_sev_info *cc_info; 196 339 ··· 217 360 bp->cc_blob_address = (u32)(unsigned long)cc_info; 218 361 219 362 return true; 220 220 - } 221 221 - 222 222 - void __head __noreturn snp_abort(void) 223 223 - { 224 224 - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 225 363 }

+15 -15

arch/x86/boot/startup/sme.c

reviewed

··· 91 91 */ 92 92 static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch"); 93 93 94 94 - static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd) 94 94 + static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd) 95 95 { 96 96 unsigned long pgd_start, pgd_end, pgd_size; 97 97 pgd_t *pgd_p; ··· 106 106 memset(pgd_p, 0, pgd_size); 107 107 } 108 108 109 109 - static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) 109 109 + static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) 110 110 { 111 111 pgd_t *pgd; 112 112 p4d_t *p4d; ··· 143 143 return pud; 144 144 } 145 145 146 146 - static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd) 146 146 + static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd) 147 147 { 148 148 pud_t *pud; 149 149 pmd_t *pmd; ··· 159 159 set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags)); 160 160 } 161 161 162 162 - static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd) 162 162 + static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd) 163 163 { 164 164 pud_t *pud; 165 165 pmd_t *pmd; ··· 185 185 set_pte(pte, __pte(ppd->paddr | ppd->pte_flags)); 186 186 } 187 187 188 188 - static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) 188 188 + static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) 189 189 { 190 190 while (ppd->vaddr < ppd->vaddr_end) { 191 191 sme_populate_pgd_large(ppd); ··· 195 195 } 196 196 } 197 197 198 198 - static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd) 198 198 + static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd) 199 199 { 200 200 while (ppd->vaddr < ppd->vaddr_end) { 201 201 sme_populate_pgd(ppd); ··· 205 205 } 206 206 } 207 207 208 208 - static void __head __sme_map_range(struct sme_populate_pgd_data *ppd, 208 208 + static void __init __sme_map_range(struct sme_populate_pgd_data *ppd, 209 209 pmdval_t pmd_flags, pteval_t pte_flags) 210 210 { 211 211 unsigned long vaddr_end; ··· 229 229 __sme_map_range_pte(ppd); 230 230 } 231 231 232 232 - static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd) 232 232 + static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd) 233 233 { 234 234 __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC); 235 235 } 236 236 237 237 - static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd) 237 237 + static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd) 238 238 { 239 239 __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC); 240 240 } 241 241 242 242 - static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd) 242 242 + static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd) 243 243 { 244 244 __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP); 245 245 } 246 246 247 247 - static unsigned long __head sme_pgtable_calc(unsigned long len) 247 247 + static unsigned long __init sme_pgtable_calc(unsigned long len) 248 248 { 249 249 unsigned long entries = 0, tables = 0; 250 250 ··· 281 281 return entries + tables; 282 282 } 283 283 284 284 - void __head sme_encrypt_kernel(struct boot_params *bp) 284 284 + void __init sme_encrypt_kernel(struct boot_params *bp) 285 285 { 286 286 unsigned long workarea_start, workarea_end, workarea_len; 287 287 unsigned long execute_start, execute_end, execute_len; ··· 485 485 native_write_cr3(__native_read_cr3()); 486 486 } 487 487 488 488 - void __head sme_enable(struct boot_params *bp) 488 488 + void __init sme_enable(struct boot_params *bp) 489 489 { 490 490 unsigned int eax, ebx, ecx, edx; 491 491 unsigned long feature_mask; ··· 521 521 return; 522 522 523 523 me_mask = 1UL << (ebx & 0x3f); 524 524 + sev_snp_needs_sfw = !(ebx & BIT(31)); 524 525 525 526 /* Check the SEV MSR whether SEV or SME is enabled */ 526 527 sev_status = msr = native_rdmsrq(MSR_AMD64_SEV); ··· 532 531 * enablement abort the guest. 533 532 */ 534 533 if (snp_en ^ !!(msr & MSR_AMD64_SEV_SNP_ENABLED)) 535 535 - snp_abort(); 534 534 + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 536 535 537 536 /* Check if memory encryption is enabled */ 538 537 if (feature_mask == AMD_SME_BIT) { ··· 568 567 569 568 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION 570 569 /* Local version for startup code, which never operates on user page tables */ 571 571 - __weak 572 570 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) 573 571 { 574 572 return pgd;

+3

arch/x86/coco/core.c

reviewed

··· 104 104 case CC_ATTR_HOST_SEV_SNP: 105 105 return cc_flags.host_sev_snp; 106 106 107 107 + case CC_ATTR_SNP_SECURE_AVIC: 108 108 + return sev_status & MSR_AMD64_SNP_SECURE_AVIC; 109 109 + 107 110 default: 108 111 return false; 109 112 }

+4 -4

arch/x86/coco/sev/Makefile

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 3 3 - obj-y += core.o sev-nmi.o vc-handle.o 3 3 + obj-y += core.o noinstr.o vc-handle.o 4 4 5 5 # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining 6 6 - UBSAN_SANITIZE_sev-nmi.o := n 6 6 + UBSAN_SANITIZE_noinstr.o := n 7 7 8 8 # GCC may fail to respect __no_sanitize_address or __no_kcsan when inlining 9 9 - KASAN_SANITIZE_sev-nmi.o := n 10 10 - KCSAN_SANITIZE_sev-nmi.o := n 9 9 + KASAN_SANITIZE_noinstr.o := n 10 10 + KCSAN_SANITIZE_noinstr.o := n

+240 -36

arch/x86/coco/sev/core.c

reviewed

··· 46 46 #include <asm/cmdline.h> 47 47 #include <asm/msr.h> 48 48 49 49 + /* Bitmap of SEV features supported by the hypervisor */ 50 50 + u64 sev_hv_features __ro_after_init; 51 51 + SYM_PIC_ALIAS(sev_hv_features); 52 52 + 53 53 + /* Secrets page physical address from the CC blob */ 54 54 + u64 sev_secrets_pa __ro_after_init; 55 55 + SYM_PIC_ALIAS(sev_secrets_pa); 56 56 + 57 57 + /* For early boot SVSM communication */ 58 58 + struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); 59 59 + SYM_PIC_ALIAS(boot_svsm_ca_page); 60 60 + 61 61 + /* 62 62 + * SVSM related information: 63 63 + * During boot, the page tables are set up as identity mapped and later 64 64 + * changed to use kernel virtual addresses. Maintain separate virtual and 65 65 + * physical addresses for the CAA to allow SVSM functions to be used during 66 66 + * early boot, both with identity mapped virtual addresses and proper kernel 67 67 + * virtual addresses. 68 68 + */ 69 69 + u64 boot_svsm_caa_pa __ro_after_init; 70 70 + SYM_PIC_ALIAS(boot_svsm_caa_pa); 71 71 + 72 72 + DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); 73 73 + DEFINE_PER_CPU(u64, svsm_caa_pa); 74 74 + 75 75 + static inline struct svsm_ca *svsm_get_caa(void) 76 76 + { 77 77 + if (sev_cfg.use_cas) 78 78 + return this_cpu_read(svsm_caa); 79 79 + else 80 80 + return rip_rel_ptr(&boot_svsm_ca_page); 81 81 + } 82 82 + 83 83 + static inline u64 svsm_get_caa_pa(void) 84 84 + { 85 85 + if (sev_cfg.use_cas) 86 86 + return this_cpu_read(svsm_caa_pa); 87 87 + else 88 88 + return boot_svsm_caa_pa; 89 89 + } 90 90 + 49 91 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */ 50 92 #define AP_INIT_CS_LIMIT 0xffff 51 93 #define AP_INIT_DS_LIMIT 0xffff ··· 121 79 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt", 122 80 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", 123 81 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", 82 82 + [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC", 124 83 }; 125 84 126 85 /* ··· 143 100 */ 144 101 u8 snp_vmpl __ro_after_init; 145 102 EXPORT_SYMBOL_GPL(snp_vmpl); 103 103 + SYM_PIC_ALIAS(snp_vmpl); 104 104 + 105 105 + /* 106 106 + * Since feature negotiation related variables are set early in the boot 107 107 + * process they must reside in the .data section so as not to be zeroed 108 108 + * out when the .bss section is later cleared. 109 109 + * 110 110 + * GHCB protocol version negotiated with the hypervisor. 111 111 + */ 112 112 + u16 ghcb_version __ro_after_init; 113 113 + SYM_PIC_ALIAS(ghcb_version); 114 114 + 115 115 + /* For early boot hypervisor communication in SEV-ES enabled guests */ 116 116 + static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); 117 117 + 118 118 + /* 119 119 + * Needs to be in the .data section because we need it NULL before bss is 120 120 + * cleared 121 121 + */ 122 122 + struct ghcb *boot_ghcb __section(".data"); 146 123 147 124 static u64 __init get_snp_jump_table_addr(void) 148 125 { ··· 213 150 __sev_put_ghcb(&state); 214 151 215 152 local_irq_restore(flags); 153 153 + 154 154 + return ret; 155 155 + } 156 156 + 157 157 + static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) 158 158 + { 159 159 + struct es_em_ctxt ctxt; 160 160 + u8 pending = 0; 161 161 + 162 162 + vc_ghcb_invalidate(ghcb); 163 163 + 164 164 + /* 165 165 + * Fill in protocol and format specifiers. This can be called very early 166 166 + * in the boot, so use rip-relative references as needed. 167 167 + */ 168 168 + ghcb->protocol_version = ghcb_version; 169 169 + ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 170 170 + 171 171 + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL); 172 172 + ghcb_set_sw_exit_info_1(ghcb, 0); 173 173 + ghcb_set_sw_exit_info_2(ghcb, 0); 174 174 + 175 175 + sev_es_wr_ghcb_msr(__pa(ghcb)); 176 176 + 177 177 + svsm_issue_call(call, &pending); 178 178 + 179 179 + if (pending) 180 180 + return -EINVAL; 181 181 + 182 182 + switch (verify_exception_info(ghcb, &ctxt)) { 183 183 + case ES_OK: 184 184 + break; 185 185 + case ES_EXCEPTION: 186 186 + vc_forward_exception(&ctxt); 187 187 + fallthrough; 188 188 + default: 189 189 + return -EINVAL; 190 190 + } 191 191 + 192 192 + return svsm_process_result_codes(call); 193 193 + } 194 194 + 195 195 + static int svsm_perform_call_protocol(struct svsm_call *call) 196 196 + { 197 197 + struct ghcb_state state; 198 198 + unsigned long flags; 199 199 + struct ghcb *ghcb; 200 200 + int ret; 201 201 + 202 202 + flags = native_local_irq_save(); 203 203 + 204 204 + if (sev_cfg.ghcbs_initialized) 205 205 + ghcb = __sev_get_ghcb(&state); 206 206 + else if (boot_ghcb) 207 207 + ghcb = boot_ghcb; 208 208 + else 209 209 + ghcb = NULL; 210 210 + 211 211 + do { 212 212 + ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) 213 213 + : __pi_svsm_perform_msr_protocol(call); 214 214 + } while (ret == -EAGAIN); 215 215 + 216 216 + if (sev_cfg.ghcbs_initialized) 217 217 + __sev_put_ghcb(&state); 218 218 + 219 219 + native_local_irq_restore(flags); 216 220 217 221 return ret; 218 222 } ··· 661 531 unsigned long vaddr_end; 662 532 663 533 /* Use the MSR protocol when a GHCB is not available. */ 664 664 - if (!boot_ghcb) 665 665 - return early_set_pages_state(vaddr, __pa(vaddr), npages, op); 534 534 + if (!boot_ghcb) { 535 535 + struct psc_desc d = { op, svsm_get_caa(), svsm_get_caa_pa() }; 536 536 + 537 537 + return early_set_pages_state(vaddr, __pa(vaddr), npages, &d); 538 538 + } 666 539 667 540 vaddr = vaddr & PAGE_MASK; 668 541 vaddr_end = vaddr + (npages << PAGE_SHIFT); ··· 1106 973 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT; 1107 974 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT; 1108 975 976 976 + if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) 977 977 + vmsa->vintr_ctrl |= V_GIF_MASK | V_NMI_ENABLE_MASK; 978 978 + 1109 979 /* SVME must be set. */ 1110 980 vmsa->efer = EFER_SVME; 1111 981 ··· 1243 1107 return 0; 1244 1108 } 1245 1109 1110 1110 + u64 savic_ghcb_msr_read(u32 reg) 1111 1111 + { 1112 1112 + u64 msr = APIC_BASE_MSR + (reg >> 4); 1113 1113 + struct pt_regs regs = { .cx = msr }; 1114 1114 + struct es_em_ctxt ctxt = { .regs = &regs }; 1115 1115 + struct ghcb_state state; 1116 1116 + enum es_result res; 1117 1117 + struct ghcb *ghcb; 1118 1118 + 1119 1119 + guard(irqsave)(); 1120 1120 + 1121 1121 + ghcb = __sev_get_ghcb(&state); 1122 1122 + vc_ghcb_invalidate(ghcb); 1123 1123 + 1124 1124 + res = sev_es_ghcb_handle_msr(ghcb, &ctxt, false); 1125 1125 + if (res != ES_OK) { 1126 1126 + pr_err("Secure AVIC MSR (0x%llx) read returned error (%d)\n", msr, res); 1127 1127 + /* MSR read failures are treated as fatal errors */ 1128 1128 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 1129 1129 + } 1130 1130 + 1131 1131 + __sev_put_ghcb(&state); 1132 1132 + 1133 1133 + return regs.ax | regs.dx << 32; 1134 1134 + } 1135 1135 + 1136 1136 + void savic_ghcb_msr_write(u32 reg, u64 value) 1137 1137 + { 1138 1138 + u64 msr = APIC_BASE_MSR + (reg >> 4); 1139 1139 + struct pt_regs regs = { 1140 1140 + .cx = msr, 1141 1141 + .ax = lower_32_bits(value), 1142 1142 + .dx = upper_32_bits(value) 1143 1143 + }; 1144 1144 + struct es_em_ctxt ctxt = { .regs = &regs }; 1145 1145 + struct ghcb_state state; 1146 1146 + enum es_result res; 1147 1147 + struct ghcb *ghcb; 1148 1148 + 1149 1149 + guard(irqsave)(); 1150 1150 + 1151 1151 + ghcb = __sev_get_ghcb(&state); 1152 1152 + vc_ghcb_invalidate(ghcb); 1153 1153 + 1154 1154 + res = sev_es_ghcb_handle_msr(ghcb, &ctxt, true); 1155 1155 + if (res != ES_OK) { 1156 1156 + pr_err("Secure AVIC MSR (0x%llx) write returned error (%d)\n", msr, res); 1157 1157 + /* MSR writes should never fail. Any failure is fatal error for SNP guest */ 1158 1158 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 1159 1159 + } 1160 1160 + 1161 1161 + __sev_put_ghcb(&state); 1162 1162 + } 1163 1163 + 1164 1164 + enum es_result savic_register_gpa(u64 gpa) 1165 1165 + { 1166 1166 + struct ghcb_state state; 1167 1167 + struct es_em_ctxt ctxt; 1168 1168 + enum es_result res; 1169 1169 + struct ghcb *ghcb; 1170 1170 + 1171 1171 + guard(irqsave)(); 1172 1172 + 1173 1173 + ghcb = __sev_get_ghcb(&state); 1174 1174 + vc_ghcb_invalidate(ghcb); 1175 1175 + 1176 1176 + ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA); 1177 1177 + ghcb_set_rbx(ghcb, gpa); 1178 1178 + res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC, 1179 1179 + SVM_VMGEXIT_SAVIC_REGISTER_GPA, 0); 1180 1180 + 1181 1181 + __sev_put_ghcb(&state); 1182 1182 + 1183 1183 + return res; 1184 1184 + } 1185 1185 + 1186 1186 + enum es_result savic_unregister_gpa(u64 *gpa) 1187 1187 + { 1188 1188 + struct ghcb_state state; 1189 1189 + struct es_em_ctxt ctxt; 1190 1190 + enum es_result res; 1191 1191 + struct ghcb *ghcb; 1192 1192 + 1193 1193 + guard(irqsave)(); 1194 1194 + 1195 1195 + ghcb = __sev_get_ghcb(&state); 1196 1196 + vc_ghcb_invalidate(ghcb); 1197 1197 + 1198 1198 + ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA); 1199 1199 + res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC, 1200 1200 + SVM_VMGEXIT_SAVIC_UNREGISTER_GPA, 0); 1201 1201 + if (gpa && res == ES_OK) 1202 1202 + *gpa = ghcb->save.rbx; 1203 1203 + 1204 1204 + __sev_put_ghcb(&state); 1205 1205 + 1206 1206 + return res; 1207 1207 + } 1208 1208 + 1246 1209 static void snp_register_per_cpu_ghcb(void) 1247 1210 { 1248 1211 struct sev_es_runtime_data *data; ··· 1468 1233 struct svsm_ca *caa; 1469 1234 1470 1235 /* Allocate the SVSM CA page if an SVSM is present */ 1471 1471 - caa = memblock_alloc_or_panic(sizeof(*caa), PAGE_SIZE); 1236 1236 + caa = cpu ? memblock_alloc_or_panic(sizeof(*caa), PAGE_SIZE) 1237 1237 + : &boot_svsm_ca_page; 1472 1238 1473 1239 per_cpu(svsm_caa, cpu) = caa; 1474 1240 per_cpu(svsm_caa_pa, cpu) = __pa(caa); ··· 1523 1287 init_ghcb(cpu); 1524 1288 } 1525 1289 1526 1526 - /* If running under an SVSM, switch to the per-cpu CA */ 1527 1527 - if (snp_vmpl) { 1528 1528 - struct svsm_call call = {}; 1529 1529 - unsigned long flags; 1530 1530 - int ret; 1531 1531 - 1532 1532 - local_irq_save(flags); 1533 1533 - 1534 1534 - /* 1535 1535 - * SVSM_CORE_REMAP_CA call: 1536 1536 - * RAX = 0 (Protocol=0, CallID=0) 1537 1537 - * RCX = New CA GPA 1538 1538 - */ 1539 1539 - call.caa = svsm_get_caa(); 1540 1540 - call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 1541 1541 - call.rcx = this_cpu_read(svsm_caa_pa); 1542 1542 - ret = svsm_perform_call_protocol(&call); 1543 1543 - if (ret) 1544 1544 - panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", 1545 1545 - ret, call.rax_out); 1546 1546 - 1290 1290 + if (snp_vmpl) 1547 1291 sev_cfg.use_cas = true; 1548 1548 - 1549 1549 - local_irq_restore(flags); 1550 1550 - } 1551 1292 1552 1293 sev_es_setup_play_dead(); 1553 1294 ··· 1801 1588 } 1802 1589 } 1803 1590 pr_cont("\n"); 1804 1804 - } 1805 1805 - 1806 1806 - void __init snp_update_svsm_ca(void) 1807 1807 - { 1808 1808 - if (!snp_vmpl) 1809 1809 - return; 1810 1810 - 1811 1811 - /* Update the CAA to a proper kernel address */ 1812 1812 - boot_svsm_caa = &boot_svsm_ca_page; 1813 1591 } 1814 1592 1815 1593 #ifdef CONFIG_SYSFS

+74

arch/x86/coco/sev/sev-nmi.c arch/x86/coco/sev/noinstr.c

reviewed

··· 106 106 107 107 __sev_put_ghcb(&state); 108 108 } 109 109 + 110 110 + /* 111 111 + * Nothing shall interrupt this code path while holding the per-CPU 112 112 + * GHCB. The backup GHCB is only for NMIs interrupting this path. 113 113 + * 114 114 + * Callers must disable local interrupts around it. 115 115 + */ 116 116 + noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) 117 117 + { 118 118 + struct sev_es_runtime_data *data; 119 119 + struct ghcb *ghcb; 120 120 + 121 121 + WARN_ON(!irqs_disabled()); 122 122 + 123 123 + data = this_cpu_read(runtime_data); 124 124 + ghcb = &data->ghcb_page; 125 125 + 126 126 + if (unlikely(data->ghcb_active)) { 127 127 + /* GHCB is already in use - save its contents */ 128 128 + 129 129 + if (unlikely(data->backup_ghcb_active)) { 130 130 + /* 131 131 + * Backup-GHCB is also already in use. There is no way 132 132 + * to continue here so just kill the machine. To make 133 133 + * panic() work, mark GHCBs inactive so that messages 134 134 + * can be printed out. 135 135 + */ 136 136 + data->ghcb_active = false; 137 137 + data->backup_ghcb_active = false; 138 138 + 139 139 + instrumentation_begin(); 140 140 + panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); 141 141 + instrumentation_end(); 142 142 + } 143 143 + 144 144 + /* Mark backup_ghcb active before writing to it */ 145 145 + data->backup_ghcb_active = true; 146 146 + 147 147 + state->ghcb = &data->backup_ghcb; 148 148 + 149 149 + /* Backup GHCB content */ 150 150 + *state->ghcb = *ghcb; 151 151 + } else { 152 152 + state->ghcb = NULL; 153 153 + data->ghcb_active = true; 154 154 + } 155 155 + 156 156 + return ghcb; 157 157 + } 158 158 + 159 159 + noinstr void __sev_put_ghcb(struct ghcb_state *state) 160 160 + { 161 161 + struct sev_es_runtime_data *data; 162 162 + struct ghcb *ghcb; 163 163 + 164 164 + WARN_ON(!irqs_disabled()); 165 165 + 166 166 + data = this_cpu_read(runtime_data); 167 167 + ghcb = &data->ghcb_page; 168 168 + 169 169 + if (state->ghcb) { 170 170 + /* Restore GHCB from Backup */ 171 171 + *ghcb = *state->ghcb; 172 172 + data->backup_ghcb_active = false; 173 173 + state->ghcb = NULL; 174 174 + } else { 175 175 + /* 176 176 + * Invalidate the GHCB so a VMGEXIT instruction issued 177 177 + * from userspace won't appear to be valid. 178 178 + */ 179 179 + vc_ghcb_invalidate(ghcb); 180 180 + data->ghcb_active = false; 181 181 + } 182 182 + }

+17 -5

arch/x86/coco/sev/vc-handle.c

reviewed

··· 351 351 } 352 352 353 353 #define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__) 354 354 + #define error(v) 355 355 + #define has_cpuflag(f) boot_cpu_has(f) 354 356 355 357 #include "vc-shared.c" 356 358 ··· 404 402 return ES_OK; 405 403 } 406 404 407 407 - static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 405 405 + enum es_result sev_es_ghcb_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt, bool write) 408 406 { 409 407 struct pt_regs *regs = ctxt->regs; 410 408 enum es_result ret; 411 411 - bool write; 412 412 - 413 413 - /* Is it a WRMSR? */ 414 414 - write = ctxt->insn.opcode.bytes[1] == 0x30; 415 409 416 410 switch (regs->cx) { 417 411 case MSR_SVSM_CAA: ··· 416 418 case MSR_AMD64_GUEST_TSC_FREQ: 417 419 if (sev_status & MSR_AMD64_SNP_SECURE_TSC) 418 420 return __vc_handle_secure_tsc_msrs(ctxt, write); 421 421 + break; 422 422 + case MSR_AMD64_SAVIC_CONTROL: 423 423 + /* 424 424 + * AMD64_SAVIC_CONTROL should not be intercepted when 425 425 + * Secure AVIC is enabled. Terminate the Secure AVIC guest 426 426 + * if the interception is enabled. 427 427 + */ 428 428 + if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) 429 429 + return ES_VMM_ERROR; 419 430 break; 420 431 default: 421 432 break; ··· 444 437 } 445 438 446 439 return ret; 440 440 + } 441 441 + 442 442 + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 443 443 + { 444 444 + return sev_es_ghcb_handle_msr(ghcb, ctxt, ctxt->insn.opcode.bytes[1] == 0x30); 447 445 } 448 446 449 447 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)

+142 -1

arch/x86/coco/sev/vc-shared.c

reviewed

··· 409 409 return ret; 410 410 } 411 411 412 412 + enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 413 413 + { 414 414 + u32 ret; 415 415 + 416 416 + ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 417 417 + if (!ret) 418 418 + return ES_OK; 419 419 + 420 420 + if (ret == 1) { 421 421 + u64 info = ghcb->save.sw_exit_info_2; 422 422 + unsigned long v = info & SVM_EVTINJ_VEC_MASK; 423 423 + 424 424 + /* Check if exception information from hypervisor is sane. */ 425 425 + if ((info & SVM_EVTINJ_VALID) && 426 426 + ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 427 427 + ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 428 428 + ctxt->fi.vector = v; 429 429 + 430 430 + if (info & SVM_EVTINJ_VALID_ERR) 431 431 + ctxt->fi.error_code = info >> 32; 432 432 + 433 433 + return ES_EXCEPTION; 434 434 + } 435 435 + } 436 436 + 437 437 + return ES_VMM_ERROR; 438 438 + } 439 439 + 440 440 + enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 441 441 + struct es_em_ctxt *ctxt, 442 442 + u64 exit_code, u64 exit_info_1, 443 443 + u64 exit_info_2) 444 444 + { 445 445 + /* Fill in protocol and format specifiers */ 446 446 + ghcb->protocol_version = ghcb_version; 447 447 + ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 448 448 + 449 449 + ghcb_set_sw_exit_code(ghcb, exit_code); 450 450 + ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 451 451 + ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 452 452 + 453 453 + sev_es_wr_ghcb_msr(__pa(ghcb)); 454 454 + VMGEXIT(); 455 455 + 456 456 + return verify_exception_info(ghcb, ctxt); 457 457 + } 458 458 + 459 459 + static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 460 460 + { 461 461 + u32 cr4 = native_read_cr4(); 462 462 + int ret; 463 463 + 464 464 + ghcb_set_rax(ghcb, leaf->fn); 465 465 + ghcb_set_rcx(ghcb, leaf->subfn); 466 466 + 467 467 + if (cr4 & X86_CR4_OSXSAVE) 468 468 + /* Safe to read xcr0 */ 469 469 + ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 470 470 + else 471 471 + /* xgetbv will cause #UD - use reset value for xcr0 */ 472 472 + ghcb_set_xcr0(ghcb, 1); 473 473 + 474 474 + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 475 475 + if (ret != ES_OK) 476 476 + return ret; 477 477 + 478 478 + if (!(ghcb_rax_is_valid(ghcb) && 479 479 + ghcb_rbx_is_valid(ghcb) && 480 480 + ghcb_rcx_is_valid(ghcb) && 481 481 + ghcb_rdx_is_valid(ghcb))) 482 482 + return ES_VMM_ERROR; 483 483 + 484 484 + leaf->eax = ghcb->save.rax; 485 485 + leaf->ebx = ghcb->save.rbx; 486 486 + leaf->ecx = ghcb->save.rcx; 487 487 + leaf->edx = ghcb->save.rdx; 488 488 + 489 489 + return ES_OK; 490 490 + } 491 491 + 492 492 + struct cpuid_ctx { 493 493 + struct ghcb *ghcb; 494 494 + struct es_em_ctxt *ctxt; 495 495 + }; 496 496 + 497 497 + static void snp_cpuid_hv_ghcb(void *p, struct cpuid_leaf *leaf) 498 498 + { 499 499 + struct cpuid_ctx *ctx = p; 500 500 + 501 501 + if (__sev_cpuid_hv_ghcb(ctx->ghcb, ctx->ctxt, leaf)) 502 502 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 503 503 + } 504 504 + 412 505 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 413 506 { 507 507 + struct cpuid_ctx ctx = { ghcb, ctxt }; 414 508 struct pt_regs *regs = ctxt->regs; 415 509 struct cpuid_leaf leaf; 416 510 int ret; 417 511 418 512 leaf.fn = regs->ax; 419 513 leaf.subfn = regs->cx; 420 420 - ret = snp_cpuid(ghcb, ctxt, &leaf); 514 514 + ret = snp_cpuid(snp_cpuid_hv_ghcb, &ctx, &leaf); 421 515 if (!ret) { 422 516 regs->ax = leaf.eax; 423 517 regs->bx = leaf.ebx; ··· 595 501 ctxt->regs->cx = ghcb->save.rcx; 596 502 597 503 return ES_OK; 504 504 + } 505 505 + 506 506 + void snp_register_ghcb_early(unsigned long paddr) 507 507 + { 508 508 + unsigned long pfn = paddr >> PAGE_SHIFT; 509 509 + u64 val; 510 510 + 511 511 + sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 512 512 + VMGEXIT(); 513 513 + 514 514 + val = sev_es_rd_ghcb_msr(); 515 515 + 516 516 + /* If the response GPA is not ours then abort the guest */ 517 517 + if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 518 518 + (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 519 519 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 520 520 + } 521 521 + 522 522 + bool __init sev_es_check_cpu_features(void) 523 523 + { 524 524 + if (!has_cpuflag(X86_FEATURE_RDRAND)) { 525 525 + error("RDRAND instruction not supported - no trusted source of randomness available\n"); 526 526 + return false; 527 527 + } 528 528 + 529 529 + return true; 530 530 + } 531 531 + 532 532 + bool sev_es_negotiate_protocol(void) 533 533 + { 534 534 + u64 val; 535 535 + 536 536 + /* Do the GHCB protocol version negotiation */ 537 537 + sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 538 538 + VMGEXIT(); 539 539 + val = sev_es_rd_ghcb_msr(); 540 540 + 541 541 + if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 542 542 + return false; 543 543 + 544 544 + if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 545 545 + GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 546 546 + return false; 547 547 + 548 548 + ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 549 549 + 550 550 + return true; 598 551 }

+11

arch/x86/include/asm/apic.h

reviewed

··· 305 305 306 306 /* Probe, setup and smpboot functions */ 307 307 int (*probe)(void); 308 308 + void (*setup)(void); 309 309 + void (*teardown)(void); 308 310 int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); 309 311 310 312 void (*init_apic_ldr)(void); ··· 318 316 int (*wakeup_secondary_cpu)(u32 apicid, unsigned long start_eip, unsigned int cpu); 319 317 /* wakeup secondary CPU using 64-bit wakeup point */ 320 318 int (*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip, unsigned int cpu); 319 319 + 320 320 + void (*update_vector)(unsigned int cpu, unsigned int vector, bool set); 321 321 322 322 char *name; 323 323 }; ··· 474 470 return apic_id <= apic->max_apic_id; 475 471 } 476 472 473 473 + static __always_inline void apic_update_vector(unsigned int cpu, unsigned int vector, bool set) 474 474 + { 475 475 + if (apic->update_vector) 476 476 + apic->update_vector(cpu, vector, set); 477 477 + } 478 478 + 477 479 #else /* CONFIG_X86_LOCAL_APIC */ 478 480 479 481 static inline u32 apic_read(u32 reg) { return 0; } ··· 491 481 static inline u32 safe_apic_wait_icr_idle(void) { return 0; } 492 482 static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); } 493 483 static inline void apic_setup_apic_calls(void) { } 484 484 + static inline void apic_update_vector(unsigned int cpu, unsigned int vector, bool set) { } 494 485 495 486 #define apic_update_callback(_callback, _fn) do { } while (0) 496 487

+2

arch/x86/include/asm/apicdef.h

reviewed

··· 135 135 #define APIC_TDR_DIV_128 0xA 136 136 #define APIC_EFEAT 0x400 137 137 #define APIC_ECTRL 0x410 138 138 + #define APIC_SEOI 0x420 139 139 + #define APIC_IER 0x480 138 140 #define APIC_EILVTn(n) (0x500 + 0x10 * n) 139 141 #define APIC_EILVT_NR_AMD_K8 1 /* # of extended interrupts */ 140 142 #define APIC_EILVT_NR_AMD_10H 4

+2

arch/x86/include/asm/boot.h

reviewed

··· 82 82 #ifndef __ASSEMBLER__ 83 83 extern unsigned int output_len; 84 84 extern const unsigned long kernel_text_size; 85 85 + extern const unsigned long kernel_inittext_offset; 86 86 + extern const unsigned long kernel_inittext_size; 85 87 extern const unsigned long kernel_total_size; 86 88 87 89 unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,

-6

arch/x86/include/asm/init.h

reviewed

··· 2 2 #ifndef _ASM_X86_INIT_H 3 3 #define _ASM_X86_INIT_H 4 4 5 5 - #if defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 170000 6 6 - #define __head __section(".head.text") __no_sanitize_undefined __no_stack_protector 7 7 - #else 8 8 - #define __head __section(".head.text") __no_sanitize_undefined __no_kstack_erase 9 9 - #endif 10 10 - 11 5 struct x86_mapping_info { 12 6 void *(*alloc_pgt_page)(void *); /* allocate buf for page table */ 13 7 void (*free_pgt_page)(void *, void *); /* free buf for page table */

+8 -1

arch/x86/include/asm/msr-index.h

reviewed

··· 706 706 #define MSR_AMD64_SNP_VMSA_REG_PROT BIT_ULL(MSR_AMD64_SNP_VMSA_REG_PROT_BIT) 707 707 #define MSR_AMD64_SNP_SMT_PROT_BIT 17 708 708 #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) 709 709 - #define MSR_AMD64_SNP_RESV_BIT 18 709 709 + #define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 710 710 + #define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) 711 711 + #define MSR_AMD64_SNP_RESV_BIT 19 710 712 #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT) 713 713 + #define MSR_AMD64_SAVIC_CONTROL 0xc0010138 714 714 + #define MSR_AMD64_SAVIC_EN_BIT 0 715 715 + #define MSR_AMD64_SAVIC_EN BIT_ULL(MSR_AMD64_SAVIC_EN_BIT) 716 716 + #define MSR_AMD64_SAVIC_ALLOWEDNMI_BIT 1 717 717 + #define MSR_AMD64_SAVIC_ALLOWEDNMI BIT_ULL(MSR_AMD64_SAVIC_ALLOWEDNMI_BIT) 711 718 #define MSR_AMD64_RMP_BASE 0xc0010132 712 719 #define MSR_AMD64_RMP_END 0xc0010133 713 720 #define MSR_AMD64_RMP_CFG 0xc0010136

+1

arch/x86/include/asm/setup.h

reviewed

··· 53 53 extern unsigned long __startup_64(unsigned long p2v_offset, struct boot_params *bp); 54 54 extern void startup_64_setup_gdt_idt(void); 55 55 extern void startup_64_load_idt(void *vc_handler); 56 56 + extern void __pi_startup_64_load_idt(void *vc_handler); 56 57 extern void early_setup_idt(void); 57 58 extern void __init do_early_exception(struct pt_regs *regs, int trapnr); 58 59

+1

arch/x86/include/asm/sev-common.h

reviewed

··· 208 208 #define GHCB_TERM_SVSM_CAA 9 /* SVSM is present but CAA is not page aligned */ 209 209 #define GHCB_TERM_SECURE_TSC 10 /* Secure TSC initialization failed */ 210 210 #define GHCB_TERM_SVSM_CA_REMAP_FAIL 11 /* SVSM is present but CA could not be remapped */ 211 211 + #define GHCB_TERM_SAVIC_FAIL 12 /* Secure AVIC-specific failure */ 211 212 212 213 #define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK) 213 214

+5 -23

arch/x86/include/asm/sev-internal.h

reviewed

··· 2 2 3 3 #define DR7_RESET_VALUE 0x400 4 4 5 5 - extern struct ghcb boot_ghcb_page; 6 5 extern u64 sev_hv_features; 7 6 extern u64 sev_secrets_pa; 8 7 ··· 55 56 DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa); 56 57 57 58 void early_set_pages_state(unsigned long vaddr, unsigned long paddr, 58 58 - unsigned long npages, enum psc_op op); 59 59 + unsigned long npages, const struct psc_desc *desc); 59 60 60 61 DECLARE_PER_CPU(struct svsm_ca *, svsm_caa); 61 62 DECLARE_PER_CPU(u64, svsm_caa_pa); 62 63 63 63 - extern struct svsm_ca *boot_svsm_caa; 64 64 extern u64 boot_svsm_caa_pa; 65 65 66 66 - static __always_inline struct svsm_ca *svsm_get_caa(void) 67 67 - { 68 68 - if (sev_cfg.use_cas) 69 69 - return this_cpu_read(svsm_caa); 70 70 - else 71 71 - return boot_svsm_caa; 72 72 - } 73 73 - 74 74 - static __always_inline u64 svsm_get_caa_pa(void) 75 75 - { 76 76 - if (sev_cfg.use_cas) 77 77 - return this_cpu_read(svsm_caa_pa); 78 78 - else 79 79 - return boot_svsm_caa_pa; 80 80 - } 81 81 - 82 82 - int svsm_perform_call_protocol(struct svsm_call *call); 66 66 + enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt); 67 67 + void vc_forward_exception(struct es_em_ctxt *ctxt); 83 68 84 69 static inline u64 sev_es_rd_ghcb_msr(void) 85 70 { ··· 80 97 native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); 81 98 } 82 99 83 83 - void snp_register_ghcb_early(unsigned long paddr); 84 84 - bool sev_es_negotiate_protocol(void); 85 85 - bool sev_es_check_cpu_features(void); 100 100 + enum es_result sev_es_ghcb_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt, bool write); 101 101 + 86 102 u64 get_hv_features(void); 87 103 88 104 const struct snp_cpuid_table *snp_cpuid_get_table(void);

+33 -8

arch/x86/include/asm/sev.h

reviewed

··· 503 503 } 504 504 505 505 void setup_ghcb(void); 506 506 + void snp_register_ghcb_early(unsigned long paddr); 506 507 void early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 507 508 unsigned long npages); 508 509 void early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, ··· 512 511 void snp_set_memory_private(unsigned long vaddr, unsigned long npages); 513 512 void snp_set_wakeup_secondary_cpu(void); 514 513 bool snp_init(struct boot_params *bp); 515 515 - void __noreturn snp_abort(void); 516 514 void snp_dmi_setup(void); 517 515 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input); 518 516 void snp_accept_memory(phys_addr_t start, phys_addr_t end); 519 517 u64 snp_get_unsupported_features(u64 status); 520 518 u64 sev_get_status(void); 521 519 void sev_show_status(void); 522 522 - void snp_update_svsm_ca(void); 523 520 int prepare_pte_enc(struct pte_enc_desc *d); 524 521 void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot); 525 522 void snp_kexec_finish(void); ··· 532 533 533 534 void __init snp_secure_tsc_prepare(void); 534 535 void __init snp_secure_tsc_init(void); 536 536 + enum es_result savic_register_gpa(u64 gpa); 537 537 + enum es_result savic_unregister_gpa(u64 *gpa); 538 538 + u64 savic_ghcb_msr_read(u32 reg); 539 539 + void savic_ghcb_msr_write(u32 reg, u64 value); 535 540 536 541 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 537 542 { 538 543 ghcb->save.sw_exit_code = 0; 539 544 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 540 545 } 541 541 - 542 542 - void vc_forward_exception(struct es_em_ctxt *ctxt); 543 546 544 547 /* I/O parameters for CPUID-related helpers */ 545 548 struct cpuid_leaf { ··· 553 552 u32 edx; 554 553 }; 555 554 556 556 - int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf); 555 555 + int svsm_perform_msr_protocol(struct svsm_call *call); 556 556 + int __pi_svsm_perform_msr_protocol(struct svsm_call *call); 557 557 + int snp_cpuid(void (*cpuid_fn)(void *ctx, struct cpuid_leaf *leaf), 558 558 + void *ctx, struct cpuid_leaf *leaf); 559 559 + 560 560 + void svsm_issue_call(struct svsm_call *call, u8 *pending); 561 561 + int svsm_process_result_codes(struct svsm_call *call); 557 562 558 563 void __noreturn sev_es_terminate(unsigned int set, unsigned int reason); 559 564 enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, ··· 567 560 u64 exit_code, u64 exit_info_1, 568 561 u64 exit_info_2); 569 562 563 563 + bool sev_es_negotiate_protocol(void); 564 564 + bool sev_es_check_cpu_features(void); 565 565 + 566 566 + extern u16 ghcb_version; 570 567 extern struct ghcb *boot_ghcb; 568 568 + extern bool sev_snp_needs_sfw; 569 569 + 570 570 + struct psc_desc { 571 571 + enum psc_op op; 572 572 + struct svsm_ca *ca; 573 573 + u64 caa_pa; 574 574 + }; 571 575 572 576 static inline void sev_evict_cache(void *va, int npages) 573 577 { ··· 618 600 static inline void snp_set_memory_private(unsigned long vaddr, unsigned long npages) { } 619 601 static inline void snp_set_wakeup_secondary_cpu(void) { } 620 602 static inline bool snp_init(struct boot_params *bp) { return false; } 621 621 - static inline void snp_abort(void) { } 622 603 static inline void snp_dmi_setup(void) { } 623 604 static inline int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input) 624 605 { ··· 627 610 static inline u64 snp_get_unsupported_features(u64 status) { return 0; } 628 611 static inline u64 sev_get_status(void) { return 0; } 629 612 static inline void sev_show_status(void) { } 630 630 - static inline void snp_update_svsm_ca(void) { } 631 613 static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; } 632 614 static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { } 633 615 static inline void snp_kexec_finish(void) { } ··· 640 624 static inline void __init snp_secure_tsc_prepare(void) { } 641 625 static inline void __init snp_secure_tsc_init(void) { } 642 626 static inline void sev_evict_cache(void *va, int npages) {} 627 627 + static inline enum es_result savic_register_gpa(u64 gpa) { return ES_UNSUPPORTED; } 628 628 + static inline enum es_result savic_unregister_gpa(u64 *gpa) { return ES_UNSUPPORTED; } 629 629 + static inline void savic_ghcb_msr_write(u32 reg, u64 value) { } 630 630 + static inline u64 savic_ghcb_msr_read(u32 reg) { return 0; } 643 631 644 632 #endif /* CONFIG_AMD_MEM_ENCRYPT */ 645 633 ··· 655 635 int psmash(u64 pfn); 656 636 int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable); 657 637 int rmp_make_shared(u64 pfn, enum pg_level level); 658 658 - void snp_leak_pages(u64 pfn, unsigned int npages); 638 638 + void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp); 659 639 void kdump_sev_callback(void); 660 640 void snp_fixup_e820_tables(void); 641 641 + static inline void snp_leak_pages(u64 pfn, unsigned int pages) 642 642 + { 643 643 + __snp_leak_pages(pfn, pages, true); 644 644 + } 661 645 #else 662 646 static inline bool snp_probe_rmptable_info(void) { return false; } 663 647 static inline int snp_rmptable_init(void) { return -ENOSYS; } ··· 674 650 return -ENODEV; 675 651 } 676 652 static inline int rmp_make_shared(u64 pfn, enum pg_level level) { return -ENODEV; } 653 653 + static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) {} 677 654 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} 678 655 static inline void kdump_sev_callback(void) { } 679 656 static inline void snp_fixup_e820_tables(void) {}

+4

arch/x86/include/uapi/asm/svm.h

reviewed

··· 118 118 #define SVM_VMGEXIT_AP_CREATE 1 119 119 #define SVM_VMGEXIT_AP_DESTROY 2 120 120 #define SVM_VMGEXIT_SNP_RUN_VMPL 0x80000018 121 121 + #define SVM_VMGEXIT_SAVIC 0x8000001a 122 122 + #define SVM_VMGEXIT_SAVIC_REGISTER_GPA 0 123 123 + #define SVM_VMGEXIT_SAVIC_UNREGISTER_GPA 1 124 124 + #define SVM_VMGEXIT_SAVIC_SELF_GPA ~0ULL 121 125 #define SVM_VMGEXIT_HV_FEATURES 0x8000fffd 122 126 #define SVM_VMGEXIT_TERM_REQUEST 0x8000fffe 123 127 #define SVM_VMGEXIT_TERM_REASON(reason_set, reason_code) \

+1

arch/x86/kernel/apic/Makefile

reviewed

··· 18 18 # APIC probe will depend on the listing order here 19 19 obj-$(CONFIG_X86_NUMACHIP) += apic_numachip.o 20 20 obj-$(CONFIG_X86_UV) += x2apic_uv_x.o 21 21 + obj-$(CONFIG_AMD_SECURE_AVIC) += x2apic_savic.o 21 22 obj-$(CONFIG_X86_X2APIC) += x2apic_phys.o 22 23 obj-$(CONFIG_X86_X2APIC) += x2apic_cluster.o 23 24 obj-y += apic_flat_64.o

+46 -41

arch/x86/kernel/apic/apic.c

reviewed

··· 592 592 0xF, ~0UL); 593 593 } else 594 594 clockevents_register_device(levt); 595 595 + 596 596 + apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, true); 595 597 } 596 598 597 599 /* ··· 1170 1168 if (!apic_accessible()) 1171 1169 return; 1172 1170 1171 1171 + if (apic->teardown) 1172 1172 + apic->teardown(); 1173 1173 + 1173 1174 apic_soft_disable(); 1174 1175 1175 1176 #ifdef CONFIG_X86_32 ··· 1433 1428 u32 regs[APIC_IR_REGS]; 1434 1429 }; 1435 1430 1436 1436 - static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) 1431 1431 + static bool apic_check_and_eoi_isr(union apic_ir *isr) 1437 1432 { 1438 1433 int i, bit; 1439 1439 - 1440 1440 - /* Read the IRRs */ 1441 1441 - for (i = 0; i < APIC_IR_REGS; i++) 1442 1442 - irr->regs[i] = apic_read(APIC_IRR + i * 0x10); 1443 1434 1444 1435 /* Read the ISRs */ 1445 1436 for (i = 0; i < APIC_IR_REGS; i++) 1446 1437 isr->regs[i] = apic_read(APIC_ISR + i * 0x10); 1447 1438 1448 1448 - /* 1449 1449 - * If the ISR map is not empty. ACK the APIC and run another round 1450 1450 - * to verify whether a pending IRR has been unblocked and turned 1451 1451 - * into a ISR. 1452 1452 - */ 1453 1453 - if (!bitmap_empty(isr->map, APIC_IR_BITS)) { 1454 1454 - /* 1455 1455 - * There can be multiple ISR bits set when a high priority 1456 1456 - * interrupt preempted a lower priority one. Issue an ACK 1457 1457 - * per set bit. 1458 1458 - */ 1459 1459 - for_each_set_bit(bit, isr->map, APIC_IR_BITS) 1460 1460 - apic_eoi(); 1439 1439 + /* If the ISR map empty, nothing to do here. */ 1440 1440 + if (bitmap_empty(isr->map, APIC_IR_BITS)) 1461 1441 return true; 1462 1462 - } 1463 1442 1464 1464 - return !bitmap_empty(irr->map, APIC_IR_BITS); 1443 1443 + /* 1444 1444 + * There can be multiple ISR bits set when a high priority 1445 1445 + * interrupt preempted a lower priority one. Issue an EOI for each 1446 1446 + * set bit. The priority traversal order does not matter as there 1447 1447 + * can't be new ISR bits raised at this point. What matters is that 1448 1448 + * an EOI is issued for each ISR bit. 1449 1449 + */ 1450 1450 + for_each_set_bit(bit, isr->map, APIC_IR_BITS) 1451 1451 + apic_eoi(); 1452 1452 + 1453 1453 + /* Reread the ISRs, they should be empty now */ 1454 1454 + for (i = 0; i < APIC_IR_REGS; i++) 1455 1455 + isr->regs[i] = apic_read(APIC_ISR + i * 0x10); 1456 1456 + 1457 1457 + return bitmap_empty(isr->map, APIC_IR_BITS); 1465 1458 } 1466 1459 1467 1460 /* 1468 1468 - * After a crash, we no longer service the interrupts and a pending 1469 1469 - * interrupt from previous kernel might still have ISR bit set. 1461 1461 + * If a CPU services an interrupt and crashes before issuing EOI to the 1462 1462 + * local APIC, the corresponding ISR bit is still set when the crashing CPU 1463 1463 + * jumps into a crash kernel. Read the ISR and issue an EOI for each set 1464 1464 + * bit to acknowledge it as otherwise these slots would be locked forever 1465 1465 + * waiting for an EOI. 1470 1466 * 1471 1471 - * Most probably by now the CPU has serviced that pending interrupt and it 1472 1472 - * might not have done the apic_eoi() because it thought, interrupt 1473 1473 - * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear 1474 1474 - * the ISR bit and cpu thinks it has already serviced the interrupt. Hence 1475 1475 - * a vector might get locked. It was noticed for timer irq (vector 1476 1476 - * 0x31). Issue an extra EOI to clear ISR. 1467 1467 + * If there are pending bits in the IRR, then they won't be converted into 1468 1468 + * ISR bits as the CPU has interrupts disabled. They will be delivered once 1469 1469 + * the CPU enables interrupts and there is nothing which can prevent that. 1477 1470 * 1478 1478 - * If there are pending IRR bits they turn into ISR bits after a higher 1479 1479 - * priority ISR bit has been acked. 1471 1471 + * In the worst case this results in spurious interrupt warnings. 1480 1472 */ 1481 1481 - static void apic_pending_intr_clear(void) 1473 1473 + static void apic_clear_isr(void) 1482 1474 { 1483 1483 - union apic_ir irr, isr; 1475 1475 + union apic_ir ir; 1484 1476 unsigned int i; 1485 1477 1486 1486 - /* 512 loops are way oversized and give the APIC a chance to obey. */ 1487 1487 - for (i = 0; i < 512; i++) { 1488 1488 - if (!apic_check_and_ack(&irr, &isr)) 1489 1489 - return; 1490 1490 - } 1491 1491 - /* Dump the IRR/ISR content if that failed */ 1492 1492 - pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map); 1478 1478 + if (!apic_check_and_eoi_isr(&ir)) 1479 1479 + pr_warn("APIC: Stale ISR: %256pb\n", ir.map); 1480 1480 + 1481 1481 + for (i = 0; i < APIC_IR_REGS; i++) 1482 1482 + ir.regs[i] = apic_read(APIC_IRR + i * 0x10); 1483 1483 + 1484 1484 + if (!bitmap_empty(ir.map, APIC_IR_BITS)) 1485 1485 + pr_warn("APIC: Stale IRR: %256pb\n", ir.map); 1493 1486 } 1494 1487 1495 1488 /** ··· 1505 1502 disable_ioapic_support(); 1506 1503 return; 1507 1504 } 1505 1505 + 1506 1506 + if (apic->setup) 1507 1507 + apic->setup(); 1508 1508 1509 1509 /* 1510 1510 * If this comes from kexec/kcrash the APIC might be enabled in ··· 1547 1541 value |= 0x10; 1548 1542 apic_write(APIC_TASKPRI, value); 1549 1543 1550 1550 - /* Clear eventually stale ISR/IRR bits */ 1551 1551 - apic_pending_intr_clear(); 1544 1544 + apic_clear_isr(); 1552 1545 1553 1546 /* 1554 1547 * Now that we are all set up, enable the APIC

+17 -11

arch/x86/kernel/apic/vector.c

reviewed

··· 134 134 135 135 apicd->hw_irq_cfg.vector = vector; 136 136 apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu); 137 137 + 138 138 + apic_update_vector(cpu, vector, true); 139 139 + 137 140 irq_data_update_effective_affinity(irqd, cpumask_of(cpu)); 138 138 - trace_vector_config(irqd->irq, vector, cpu, 139 139 - apicd->hw_irq_cfg.dest_apicid); 141 141 + trace_vector_config(irqd->irq, vector, cpu, apicd->hw_irq_cfg.dest_apicid); 140 142 } 141 143 142 142 - static void apic_update_vector(struct irq_data *irqd, unsigned int newvec, 143 143 - unsigned int newcpu) 144 144 + static void apic_free_vector(unsigned int cpu, unsigned int vector, bool managed) 145 145 + { 146 146 + apic_update_vector(cpu, vector, false); 147 147 + irq_matrix_free(vector_matrix, cpu, vector, managed); 148 148 + } 149 149 + 150 150 + static void chip_data_update(struct irq_data *irqd, unsigned int newvec, unsigned int newcpu) 144 151 { 145 152 struct apic_chip_data *apicd = apic_chip_data(irqd); 146 153 struct irq_desc *desc = irq_data_to_desc(irqd); ··· 181 174 apicd->prev_cpu = apicd->cpu; 182 175 WARN_ON_ONCE(apicd->cpu == newcpu); 183 176 } else { 184 184 - irq_matrix_free(vector_matrix, apicd->cpu, apicd->vector, 185 185 - managed); 177 177 + apic_free_vector(apicd->cpu, apicd->vector, managed); 186 178 } 187 179 188 180 setnew: ··· 267 261 trace_vector_alloc(irqd->irq, vector, resvd, vector); 268 262 if (vector < 0) 269 263 return vector; 270 270 - apic_update_vector(irqd, vector, cpu); 264 264 + chip_data_update(irqd, vector, cpu); 271 265 272 266 return 0; 273 267 } ··· 343 337 trace_vector_alloc_managed(irqd->irq, vector, vector); 344 338 if (vector < 0) 345 339 return vector; 346 346 - apic_update_vector(irqd, vector, cpu); 340 340 + chip_data_update(irqd, vector, cpu); 347 341 348 342 return 0; 349 343 } ··· 363 357 apicd->prev_cpu); 364 358 365 359 per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_SHUTDOWN; 366 366 - irq_matrix_free(vector_matrix, apicd->cpu, vector, managed); 360 360 + apic_free_vector(apicd->cpu, vector, managed); 367 361 apicd->vector = 0; 368 362 369 363 /* Clean up move in progress */ ··· 372 366 return; 373 367 374 368 per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_SHUTDOWN; 375 375 - irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, managed); 369 369 + apic_free_vector(apicd->prev_cpu, vector, managed); 376 370 apicd->prev_vector = 0; 377 371 apicd->move_in_progress = 0; 378 372 hlist_del_init(&apicd->clist); ··· 911 905 * affinity mask comes online. 912 906 */ 913 907 trace_vector_free_moved(apicd->irq, cpu, vector, managed); 914 914 - irq_matrix_free(vector_matrix, cpu, vector, managed); 908 908 + apic_free_vector(cpu, vector, managed); 915 909 per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED; 916 910 hlist_del_init(&apicd->clist); 917 911 apicd->prev_vector = 0;

+428

arch/x86/kernel/apic/x2apic_savic.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * AMD Secure AVIC Support (SEV-SNP Guests) 4 4 + * 5 5 + * Copyright (C) 2024 Advanced Micro Devices, Inc. 6 6 + * 7 7 + * Author: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com> 8 8 + */ 9 9 + 10 10 + #include <linux/cc_platform.h> 11 11 + #include <linux/cpumask.h> 12 12 + #include <linux/percpu-defs.h> 13 13 + #include <linux/align.h> 14 14 + 15 15 + #include <asm/apic.h> 16 16 + #include <asm/sev.h> 17 17 + 18 18 + #include "local.h" 19 19 + 20 20 + struct secure_avic_page { 21 21 + u8 regs[PAGE_SIZE]; 22 22 + } __aligned(PAGE_SIZE); 23 23 + 24 24 + static struct secure_avic_page __percpu *savic_page __ro_after_init; 25 25 + 26 26 + static int savic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 27 27 + { 28 28 + return x2apic_enabled() && cc_platform_has(CC_ATTR_SNP_SECURE_AVIC); 29 29 + } 30 30 + 31 31 + static inline void *get_reg_bitmap(unsigned int cpu, unsigned int offset) 32 32 + { 33 33 + return &per_cpu_ptr(savic_page, cpu)->regs[offset]; 34 34 + } 35 35 + 36 36 + static inline void update_vector(unsigned int cpu, unsigned int offset, 37 37 + unsigned int vector, bool set) 38 38 + { 39 39 + void *bitmap = get_reg_bitmap(cpu, offset); 40 40 + 41 41 + if (set) 42 42 + apic_set_vector(vector, bitmap); 43 43 + else 44 44 + apic_clear_vector(vector, bitmap); 45 45 + } 46 46 + 47 47 + #define SAVIC_ALLOWED_IRR 0x204 48 48 + 49 49 + /* 50 50 + * When Secure AVIC is enabled, RDMSR/WRMSR of the APIC registers 51 51 + * result in #VC exception (for non-accelerated register accesses) 52 52 + * with VMEXIT_AVIC_NOACCEL error code. The #VC exception handler 53 53 + * can read/write the x2APIC register in the guest APIC backing page. 54 54 + * 55 55 + * Since doing this would increase the latency of accessing x2APIC 56 56 + * registers, instead of doing RDMSR/WRMSR based accesses and 57 57 + * handling the APIC register reads/writes in the #VC exception handler, 58 58 + * the read() and write() callbacks directly read/write the APIC register 59 59 + * from/to the vCPU's APIC backing page. 60 60 + */ 61 61 + static u32 savic_read(u32 reg) 62 62 + { 63 63 + void *ap = this_cpu_ptr(savic_page); 64 64 + 65 65 + switch (reg) { 66 66 + case APIC_LVTT: 67 67 + case APIC_TMICT: 68 68 + case APIC_TMCCT: 69 69 + case APIC_TDCR: 70 70 + case APIC_LVTTHMR: 71 71 + case APIC_LVTPC: 72 72 + case APIC_LVT0: 73 73 + case APIC_LVT1: 74 74 + case APIC_LVTERR: 75 75 + return savic_ghcb_msr_read(reg); 76 76 + case APIC_ID: 77 77 + case APIC_LVR: 78 78 + case APIC_TASKPRI: 79 79 + case APIC_ARBPRI: 80 80 + case APIC_PROCPRI: 81 81 + case APIC_LDR: 82 82 + case APIC_SPIV: 83 83 + case APIC_ESR: 84 84 + case APIC_EFEAT: 85 85 + case APIC_ECTRL: 86 86 + case APIC_SEOI: 87 87 + case APIC_IER: 88 88 + case APIC_EILVTn(0) ... APIC_EILVTn(3): 89 89 + return apic_get_reg(ap, reg); 90 90 + case APIC_ICR: 91 91 + return (u32)apic_get_reg64(ap, reg); 92 92 + case APIC_ISR ... APIC_ISR + 0x70: 93 93 + case APIC_TMR ... APIC_TMR + 0x70: 94 94 + if (WARN_ONCE(!IS_ALIGNED(reg, 16), 95 95 + "APIC register read offset 0x%x not aligned at 16 bytes", reg)) 96 96 + return 0; 97 97 + return apic_get_reg(ap, reg); 98 98 + /* IRR and ALLOWED_IRR offset range */ 99 99 + case APIC_IRR ... APIC_IRR + 0x74: 100 100 + /* 101 101 + * Valid APIC_IRR/SAVIC_ALLOWED_IRR registers are at 16 bytes strides from 102 102 + * their respective base offset. APIC_IRRs are in the range 103 103 + * 104 104 + * (0x200, 0x210, ..., 0x270) 105 105 + * 106 106 + * while the SAVIC_ALLOWED_IRR range starts 4 bytes later, in the range 107 107 + * 108 108 + * (0x204, 0x214, ..., 0x274). 109 109 + * 110 110 + * Filter out everything else. 111 111 + */ 112 112 + if (WARN_ONCE(!(IS_ALIGNED(reg, 16) || 113 113 + IS_ALIGNED(reg - 4, 16)), 114 114 + "Misaligned APIC_IRR/ALLOWED_IRR APIC register read offset 0x%x", reg)) 115 115 + return 0; 116 116 + return apic_get_reg(ap, reg); 117 117 + default: 118 118 + pr_err("Error reading unknown Secure AVIC reg offset 0x%x\n", reg); 119 119 + return 0; 120 120 + } 121 121 + } 122 122 + 123 123 + #define SAVIC_NMI_REQ 0x278 124 124 + 125 125 + /* 126 126 + * On WRMSR to APIC_SELF_IPI register by the guest, Secure AVIC hardware 127 127 + * updates the APIC_IRR in the APIC backing page of the vCPU. In addition, 128 128 + * hardware evaluates the new APIC_IRR update for interrupt injection to 129 129 + * the vCPU. So, self IPIs are hardware-accelerated. 130 130 + */ 131 131 + static inline void self_ipi_reg_write(unsigned int vector) 132 132 + { 133 133 + native_apic_msr_write(APIC_SELF_IPI, vector); 134 134 + } 135 135 + 136 136 + static void send_ipi_dest(unsigned int cpu, unsigned int vector, bool nmi) 137 137 + { 138 138 + if (nmi) 139 139 + apic_set_reg(per_cpu_ptr(savic_page, cpu), SAVIC_NMI_REQ, 1); 140 140 + else 141 141 + update_vector(cpu, APIC_IRR, vector, true); 142 142 + } 143 143 + 144 144 + static void send_ipi_allbut(unsigned int vector, bool nmi) 145 145 + { 146 146 + unsigned int cpu, src_cpu; 147 147 + 148 148 + guard(irqsave)(); 149 149 + 150 150 + src_cpu = raw_smp_processor_id(); 151 151 + 152 152 + for_each_cpu(cpu, cpu_online_mask) { 153 153 + if (cpu == src_cpu) 154 154 + continue; 155 155 + send_ipi_dest(cpu, vector, nmi); 156 156 + } 157 157 + } 158 158 + 159 159 + static inline void self_ipi(unsigned int vector, bool nmi) 160 160 + { 161 161 + u32 icr_low = APIC_SELF_IPI | vector; 162 162 + 163 163 + if (nmi) 164 164 + icr_low |= APIC_DM_NMI; 165 165 + 166 166 + native_x2apic_icr_write(icr_low, 0); 167 167 + } 168 168 + 169 169 + static void savic_icr_write(u32 icr_low, u32 icr_high) 170 170 + { 171 171 + unsigned int dsh, vector; 172 172 + u64 icr_data; 173 173 + bool nmi; 174 174 + 175 175 + dsh = icr_low & APIC_DEST_ALLBUT; 176 176 + vector = icr_low & APIC_VECTOR_MASK; 177 177 + nmi = ((icr_low & APIC_DM_FIXED_MASK) == APIC_DM_NMI); 178 178 + 179 179 + switch (dsh) { 180 180 + case APIC_DEST_SELF: 181 181 + self_ipi(vector, nmi); 182 182 + break; 183 183 + case APIC_DEST_ALLINC: 184 184 + self_ipi(vector, nmi); 185 185 + fallthrough; 186 186 + case APIC_DEST_ALLBUT: 187 187 + send_ipi_allbut(vector, nmi); 188 188 + break; 189 189 + default: 190 190 + send_ipi_dest(icr_high, vector, nmi); 191 191 + break; 192 192 + } 193 193 + 194 194 + icr_data = ((u64)icr_high) << 32 | icr_low; 195 195 + if (dsh != APIC_DEST_SELF) 196 196 + savic_ghcb_msr_write(APIC_ICR, icr_data); 197 197 + apic_set_reg64(this_cpu_ptr(savic_page), APIC_ICR, icr_data); 198 198 + } 199 199 + 200 200 + static void savic_write(u32 reg, u32 data) 201 201 + { 202 202 + void *ap = this_cpu_ptr(savic_page); 203 203 + 204 204 + switch (reg) { 205 205 + case APIC_LVTT: 206 206 + case APIC_TMICT: 207 207 + case APIC_TDCR: 208 208 + case APIC_LVT0: 209 209 + case APIC_LVT1: 210 210 + case APIC_LVTTHMR: 211 211 + case APIC_LVTPC: 212 212 + case APIC_LVTERR: 213 213 + savic_ghcb_msr_write(reg, data); 214 214 + break; 215 215 + case APIC_TASKPRI: 216 216 + case APIC_EOI: 217 217 + case APIC_SPIV: 218 218 + case SAVIC_NMI_REQ: 219 219 + case APIC_ESR: 220 220 + case APIC_ECTRL: 221 221 + case APIC_SEOI: 222 222 + case APIC_IER: 223 223 + case APIC_EILVTn(0) ... APIC_EILVTn(3): 224 224 + apic_set_reg(ap, reg, data); 225 225 + break; 226 226 + case APIC_ICR: 227 227 + savic_icr_write(data, 0); 228 228 + break; 229 229 + case APIC_SELF_IPI: 230 230 + self_ipi_reg_write(data); 231 231 + break; 232 232 + /* ALLOWED_IRR offsets are writable */ 233 233 + case SAVIC_ALLOWED_IRR ... SAVIC_ALLOWED_IRR + 0x70: 234 234 + if (IS_ALIGNED(reg - 4, 16)) { 235 235 + apic_set_reg(ap, reg, data); 236 236 + break; 237 237 + } 238 238 + fallthrough; 239 239 + default: 240 240 + pr_err("Error writing unknown Secure AVIC reg offset 0x%x\n", reg); 241 241 + } 242 242 + } 243 243 + 244 244 + static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh) 245 245 + { 246 246 + unsigned int icr_low; 247 247 + 248 248 + icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL); 249 249 + savic_icr_write(icr_low, dest); 250 250 + } 251 251 + 252 252 + static void savic_send_ipi(int cpu, int vector) 253 253 + { 254 254 + u32 dest = per_cpu(x86_cpu_to_apicid, cpu); 255 255 + 256 256 + send_ipi(dest, vector, 0); 257 257 + } 258 258 + 259 259 + static void send_ipi_mask(const struct cpumask *mask, unsigned int vector, bool excl_self) 260 260 + { 261 261 + unsigned int cpu, this_cpu; 262 262 + 263 263 + guard(irqsave)(); 264 264 + 265 265 + this_cpu = raw_smp_processor_id(); 266 266 + 267 267 + for_each_cpu(cpu, mask) { 268 268 + if (excl_self && cpu == this_cpu) 269 269 + continue; 270 270 + send_ipi(per_cpu(x86_cpu_to_apicid, cpu), vector, 0); 271 271 + } 272 272 + } 273 273 + 274 274 + static void savic_send_ipi_mask(const struct cpumask *mask, int vector) 275 275 + { 276 276 + send_ipi_mask(mask, vector, false); 277 277 + } 278 278 + 279 279 + static void savic_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) 280 280 + { 281 281 + send_ipi_mask(mask, vector, true); 282 282 + } 283 283 + 284 284 + static void savic_send_ipi_allbutself(int vector) 285 285 + { 286 286 + send_ipi(0, vector, APIC_DEST_ALLBUT); 287 287 + } 288 288 + 289 289 + static void savic_send_ipi_all(int vector) 290 290 + { 291 291 + send_ipi(0, vector, APIC_DEST_ALLINC); 292 292 + } 293 293 + 294 294 + static void savic_send_ipi_self(int vector) 295 295 + { 296 296 + self_ipi_reg_write(vector); 297 297 + } 298 298 + 299 299 + static void savic_update_vector(unsigned int cpu, unsigned int vector, bool set) 300 300 + { 301 301 + update_vector(cpu, SAVIC_ALLOWED_IRR, vector, set); 302 302 + } 303 303 + 304 304 + static void savic_eoi(void) 305 305 + { 306 306 + unsigned int cpu; 307 307 + int vec; 308 308 + 309 309 + cpu = raw_smp_processor_id(); 310 310 + vec = apic_find_highest_vector(get_reg_bitmap(cpu, APIC_ISR)); 311 311 + if (WARN_ONCE(vec == -1, "EOI write while no active interrupt in APIC_ISR")) 312 312 + return; 313 313 + 314 314 + /* Is level-triggered interrupt? */ 315 315 + if (apic_test_vector(vec, get_reg_bitmap(cpu, APIC_TMR))) { 316 316 + update_vector(cpu, APIC_ISR, vec, false); 317 317 + /* 318 318 + * Propagate the EOI write to the hypervisor for level-triggered 319 319 + * interrupts. Return to the guest from GHCB protocol event takes 320 320 + * care of re-evaluating interrupt state. 321 321 + */ 322 322 + savic_ghcb_msr_write(APIC_EOI, 0); 323 323 + } else { 324 324 + /* 325 325 + * Hardware clears APIC_ISR and re-evaluates the interrupt state 326 326 + * to determine if there is any pending interrupt which can be 327 327 + * delivered to CPU. 328 328 + */ 329 329 + native_apic_msr_eoi(); 330 330 + } 331 331 + } 332 332 + 333 333 + static void savic_teardown(void) 334 334 + { 335 335 + /* Disable Secure AVIC */ 336 336 + native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 0); 337 337 + savic_unregister_gpa(NULL); 338 338 + } 339 339 + 340 340 + static void savic_setup(void) 341 341 + { 342 342 + void *ap = this_cpu_ptr(savic_page); 343 343 + enum es_result res; 344 344 + unsigned long gpa; 345 345 + 346 346 + /* 347 347 + * Before Secure AVIC is enabled, APIC MSR reads are intercepted. 348 348 + * APIC_ID MSR read returns the value from the hypervisor. 349 349 + */ 350 350 + apic_set_reg(ap, APIC_ID, native_apic_msr_read(APIC_ID)); 351 351 + 352 352 + gpa = __pa(ap); 353 353 + 354 354 + /* 355 355 + * The NPT entry for a vCPU's APIC backing page must always be 356 356 + * present when the vCPU is running in order for Secure AVIC to 357 357 + * function. A VMEXIT_BUSY is returned on VMRUN and the vCPU cannot 358 358 + * be resumed if the NPT entry for the APIC backing page is not 359 359 + * present. Notify GPA of the vCPU's APIC backing page to the 360 360 + * hypervisor by calling savic_register_gpa(). Before executing 361 361 + * VMRUN, the hypervisor makes use of this information to make sure 362 362 + * the APIC backing page is mapped in NPT. 363 363 + */ 364 364 + res = savic_register_gpa(gpa); 365 365 + if (res != ES_OK) 366 366 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 367 367 + 368 368 + native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 369 369 + gpa | MSR_AMD64_SAVIC_EN | MSR_AMD64_SAVIC_ALLOWEDNMI); 370 370 + } 371 371 + 372 372 + static int savic_probe(void) 373 373 + { 374 374 + if (!cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) 375 375 + return 0; 376 376 + 377 377 + if (!x2apic_mode) { 378 378 + pr_err("Secure AVIC enabled in non x2APIC mode\n"); 379 379 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 380 380 + /* unreachable */ 381 381 + } 382 382 + 383 383 + savic_page = alloc_percpu(struct secure_avic_page); 384 384 + if (!savic_page) 385 385 + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 386 386 + 387 387 + return 1; 388 388 + } 389 389 + 390 390 + static struct apic apic_x2apic_savic __ro_after_init = { 391 391 + 392 392 + .name = "secure avic x2apic", 393 393 + .probe = savic_probe, 394 394 + .acpi_madt_oem_check = savic_acpi_madt_oem_check, 395 395 + .setup = savic_setup, 396 396 + .teardown = savic_teardown, 397 397 + 398 398 + .dest_mode_logical = false, 399 399 + 400 400 + .disable_esr = 0, 401 401 + 402 402 + .cpu_present_to_apicid = default_cpu_present_to_apicid, 403 403 + 404 404 + .max_apic_id = UINT_MAX, 405 405 + .x2apic_set_max_apicid = true, 406 406 + .get_apic_id = x2apic_get_apic_id, 407 407 + 408 408 + .calc_dest_apicid = apic_default_calc_apicid, 409 409 + 410 410 + .send_IPI = savic_send_ipi, 411 411 + .send_IPI_mask = savic_send_ipi_mask, 412 412 + .send_IPI_mask_allbutself = savic_send_ipi_mask_allbutself, 413 413 + .send_IPI_allbutself = savic_send_ipi_allbutself, 414 414 + .send_IPI_all = savic_send_ipi_all, 415 415 + .send_IPI_self = savic_send_ipi_self, 416 416 + 417 417 + .nmi_to_offline_cpu = true, 418 418 + 419 419 + .read = savic_read, 420 420 + .write = savic_write, 421 421 + .eoi = savic_eoi, 422 422 + .icr_read = native_x2apic_icr_read, 423 423 + .icr_write = savic_icr_write, 424 424 + 425 425 + .update_vector = savic_update_vector, 426 426 + }; 427 427 + 428 428 + apic_driver(apic_x2apic_savic);

+4 -1

arch/x86/kernel/head64.c

reviewed

··· 52 52 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX); 53 53 54 54 unsigned int __pgtable_l5_enabled __ro_after_init; 55 55 + SYM_PIC_ALIAS(__pgtable_l5_enabled); 55 56 unsigned int pgdir_shift __ro_after_init = 39; 56 57 EXPORT_SYMBOL(pgdir_shift); 58 58 + SYM_PIC_ALIAS(pgdir_shift); 57 59 unsigned int ptrs_per_p4d __ro_after_init = 1; 58 60 EXPORT_SYMBOL(ptrs_per_p4d); 61 61 + SYM_PIC_ALIAS(ptrs_per_p4d); 59 62 60 63 unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4; 61 64 EXPORT_SYMBOL(page_offset_base); ··· 319 316 handler = vc_boot_ghcb; 320 317 } 321 318 322 322 - startup_64_load_idt(handler); 319 319 + __pi_startup_64_load_idt(handler); 323 320 }

+4 -1

arch/x86/kernel/head_32.S

reviewed

··· 61 61 * any particular GDT layout, because we load our own as soon as we 62 62 * can. 63 63 */ 64 64 - __HEAD 64 64 + __INIT 65 65 SYM_CODE_START(startup_32) 66 66 movl pa(initial_stack),%ecx 67 67 ··· 136 136 * If cpu hotplug is not supported then this code can go in init section 137 137 * which will be freed later 138 138 */ 139 139 + #ifdef CONFIG_HOTPLUG_CPU 140 140 + .text 141 141 + #endif 139 142 SYM_FUNC_START(startup_32_smp) 140 143 cld 141 144 movl $(__BOOT_DS),%eax

+5 -5

arch/x86/kernel/head_64.S

reviewed

··· 33 33 * because we need identity-mapped pages. 34 34 */ 35 35 36 36 - __HEAD 36 36 + __INIT 37 37 .code64 38 38 SYM_CODE_START_NOALIGN(startup_64) 39 39 UNWIND_HINT_END_OF_STACK ··· 71 71 xorl %edx, %edx 72 72 wrmsr 73 73 74 74 - call startup_64_setup_gdt_idt 74 74 + call __pi_startup_64_setup_gdt_idt 75 75 76 76 /* Now switch to __KERNEL_CS so IRET works reliably */ 77 77 pushq $__KERNEL_CS ··· 91 91 * subsequent code. Pass the boot_params pointer as the first argument. 92 92 */ 93 93 movq %r15, %rdi 94 94 - call sme_enable 94 94 + call __pi_sme_enable 95 95 #endif 96 96 97 97 /* Sanitize CPU configuration */ ··· 111 111 * programmed into CR3. 112 112 */ 113 113 movq %r15, %rsi 114 114 - call __startup_64 114 114 + call __pi___startup_64 115 115 116 116 /* Form the CR3 value being sure to include the CR3 modifier */ 117 117 leaq early_top_pgt(%rip), %rcx ··· 562 562 /* Call C handler */ 563 563 movq %rsp, %rdi 564 564 movq ORIG_RAX(%rsp), %rsi 565 565 - call do_vc_no_ghcb 565 565 + call __pi_do_vc_no_ghcb 566 566 567 567 /* Unwind pt_regs */ 568 568 POP_REGS

+4 -5

arch/x86/kernel/vmlinux.lds.S

reviewed

··· 160 160 161 161 } :text = 0xcccccccc 162 162 163 163 - /* bootstrapping code */ 164 164 - .head.text : AT(ADDR(.head.text) - LOAD_OFFSET) { 165 165 - HEAD_TEXT 166 166 - } :text = 0xcccccccc 167 167 - 168 163 /* End of text section, which should occupy whole number of pages */ 169 164 _etext = .; 170 165 . = ALIGN(PAGE_SIZE); ··· 222 227 */ 223 228 .altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) { 224 229 *(.altinstr_aux) 230 230 + . = ALIGN(PAGE_SIZE); 231 231 + __inittext_end = .; 225 232 } 226 233 227 234 INIT_DATA_SECTION(16) ··· 532 535 xen_elfnote_phys32_entry_value = 533 536 ABSOLUTE(xen_elfnote_phys32_entry) + ABSOLUTE(pvh_start_xen - LOAD_OFFSET); 534 537 #endif 538 538 + 539 539 + #include "../boot/startup/exports.h"

-6

arch/x86/mm/mem_encrypt_amd.c

reviewed

··· 536 536 x86_init.resources.dmi_setup = snp_dmi_setup; 537 537 } 538 538 539 539 - /* 540 540 - * Switch the SVSM CA mapping (if active) from identity mapped to 541 541 - * kernel mapped. 542 542 - */ 543 543 - snp_update_svsm_ca(); 544 544 - 545 539 if (sev_status & MSR_AMD64_SNP_SECURE_TSC) 546 540 setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); 547 541 }

+3 -3

arch/x86/mm/mem_encrypt_boot.S

reviewed

··· 16 16 17 17 .text 18 18 .code64 19 19 - SYM_FUNC_START(sme_encrypt_execute) 19 19 + SYM_FUNC_START(__pi_sme_encrypt_execute) 20 20 21 21 /* 22 22 * Entry parameters: ··· 69 69 ANNOTATE_UNRET_SAFE 70 70 ret 71 71 int3 72 72 - SYM_FUNC_END(sme_encrypt_execute) 72 72 + SYM_FUNC_END(__pi_sme_encrypt_execute) 73 73 74 74 - SYM_FUNC_START(__enc_copy) 74 74 + SYM_FUNC_START_LOCAL(__enc_copy) 75 75 ANNOTATE_NOENDBR 76 76 /* 77 77 * Routine used to encrypt memory in place.

+1 -1

arch/x86/platform/pvh/head.S

reviewed

··· 24 24 #include <asm/nospec-branch.h> 25 25 #include <xen/interface/elfnote.h> 26 26 27 27 - __HEAD 27 27 + __INIT 28 28 29 29 /* 30 30 * Entry point for PVH guests.

+1 -7

arch/x86/tools/relocs.c

reviewed

··· 740 740 static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym, 741 741 const char *symname) 742 742 { 743 743 - int headtext = !strcmp(sec_name(sec->shdr.sh_info), ".head.text"); 744 743 unsigned r_type = ELF64_R_TYPE(rel->r_info); 745 744 ElfW(Addr) offset = rel->r_offset; 746 745 int shn_abs = (sym->st_shndx == SHN_ABS) && !is_reloc(S_REL, symname); 746 746 + 747 747 if (sym->st_shndx == SHN_UNDEF) 748 748 return 0; 749 749 ··· 780 780 break; 781 781 782 782 die("Invalid absolute %s relocation: %s\n", rel_type(r_type), symname); 783 783 - break; 784 784 - } 785 785 - 786 786 - if (headtext) { 787 787 - die("Absolute reference to symbol '%s' not permitted in .head.text\n", 788 788 - symname); 789 783 break; 790 784 } 791 785

+4 -3

arch/x86/virt/svm/sev.c

reviewed

··· 1029 1029 } 1030 1030 EXPORT_SYMBOL_GPL(rmp_make_shared); 1031 1031 1032 1032 - void snp_leak_pages(u64 pfn, unsigned int npages) 1032 1032 + void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) 1033 1033 { 1034 1034 struct page *page = pfn_to_page(pfn); 1035 1035 ··· 1052 1052 (PageHead(page) && compound_nr(page) <= npages)) 1053 1053 list_add_tail(&page->buddy_list, &snp_leaked_pages_list); 1054 1054 1055 1055 - dump_rmpentry(pfn); 1055 1055 + if (dump_rmp) 1056 1056 + dump_rmpentry(pfn); 1056 1057 snp_nr_leaked_pages++; 1057 1058 pfn++; 1058 1059 page++; 1059 1060 } 1060 1061 spin_unlock(&snp_leaked_pages_list_lock); 1061 1062 } 1062 1062 - EXPORT_SYMBOL_GPL(snp_leak_pages); 1063 1063 + EXPORT_SYMBOL_GPL(__snp_leak_pages); 1063 1064 1064 1065 void kdump_sev_callback(void) 1065 1066 {

+2 -1

drivers/crypto/ccp/Makefile

reviewed

··· 13 13 tee-dev.o \ 14 14 platform-access.o \ 15 15 dbc.o \ 16 16 - hsti.o 16 16 + hsti.o \ 17 17 + sfs.o 17 18 18 19 obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o 19 20 ccp-crypto-objs := ccp-crypto-main.o \

+20

drivers/crypto/ccp/psp-dev.c

reviewed

··· 17 17 #include "psp-dev.h" 18 18 #include "sev-dev.h" 19 19 #include "tee-dev.h" 20 20 + #include "sfs.h" 20 21 #include "platform-access.h" 21 22 #include "dbc.h" 22 23 #include "hsti.h" ··· 183 182 return 0; 184 183 } 185 184 185 185 + static int psp_check_sfs_support(struct psp_device *psp) 186 186 + { 187 187 + /* Check if device supports SFS feature */ 188 188 + if (!psp->capability.sfs) { 189 189 + dev_dbg(psp->dev, "psp does not support SFS\n"); 190 190 + return -ENODEV; 191 191 + } 192 192 + 193 193 + return 0; 194 194 + } 195 195 + 186 196 static int psp_init(struct psp_device *psp) 187 197 { 188 198 int ret; ··· 206 194 207 195 if (!psp_check_tee_support(psp)) { 208 196 ret = tee_dev_init(psp); 197 197 + if (ret) 198 198 + return ret; 199 199 + } 200 200 + 201 201 + if (!psp_check_sfs_support(psp)) { 202 202 + ret = sfs_dev_init(psp); 209 203 if (ret) 210 204 return ret; 211 205 } ··· 319 301 sev_dev_destroy(psp); 320 302 321 303 tee_dev_destroy(psp); 304 304 + 305 305 + sfs_dev_destroy(psp); 322 306 323 307 dbc_dev_destroy(psp); 324 308

+7 -1

drivers/crypto/ccp/psp-dev.h

reviewed

··· 32 32 unsigned int sev :1, 33 33 tee :1, 34 34 dbc_thru_ext :1, 35 35 - rsvd1 :4, 35 35 + sfs :1, 36 36 + rsvd1 :3, 36 37 security_reporting :1, 37 38 fused_part :1, 38 39 rsvd2 :1, ··· 69 68 void *tee_data; 70 69 void *platform_access_data; 71 70 void *dbc_data; 71 71 + void *sfs_data; 72 72 73 73 union psp_cap_register capability; 74 74 }; ··· 120 118 * @PSP_SUB_CMD_DBC_SET_UID: Set UID for DBC 121 119 * @PSP_SUB_CMD_DBC_GET_PARAMETER: Get parameter from DBC 122 120 * @PSP_SUB_CMD_DBC_SET_PARAMETER: Set parameter for DBC 121 121 + * @PSP_SUB_CMD_SFS_GET_FW_VERS: Get firmware versions for ASP and other MP 122 122 + * @PSP_SUB_CMD_SFS_UPDATE: Command to load, verify and execute SFS package 123 123 */ 124 124 enum psp_sub_cmd { 125 125 PSP_SUB_CMD_DBC_GET_NONCE = PSP_DYNAMIC_BOOST_GET_NONCE, 126 126 PSP_SUB_CMD_DBC_SET_UID = PSP_DYNAMIC_BOOST_SET_UID, 127 127 PSP_SUB_CMD_DBC_GET_PARAMETER = PSP_DYNAMIC_BOOST_GET_PARAMETER, 128 128 PSP_SUB_CMD_DBC_SET_PARAMETER = PSP_DYNAMIC_BOOST_SET_PARAMETER, 129 129 + PSP_SUB_CMD_SFS_GET_FW_VERS = PSP_SFS_GET_FW_VERSIONS, 130 130 + PSP_SUB_CMD_SFS_UPDATE = PSP_SFS_UPDATE, 129 131 }; 130 132 131 133 int psp_extended_mailbox_cmd(struct psp_device *psp, unsigned int timeout_msecs,

+182

drivers/crypto/ccp/sev-dev.c

reviewed

··· 82 82 static bool psp_dead; 83 83 static int psp_timeout; 84 84 85 85 + enum snp_hv_fixed_pages_state { 86 86 + ALLOCATED, 87 87 + HV_FIXED, 88 88 + }; 89 89 + 90 90 + struct snp_hv_fixed_pages_entry { 91 91 + struct list_head list; 92 92 + struct page *page; 93 93 + unsigned int order; 94 94 + bool free; 95 95 + enum snp_hv_fixed_pages_state page_state; 96 96 + }; 97 97 + 98 98 + static LIST_HEAD(snp_hv_fixed_pages); 99 99 + 85 100 /* Trusted Memory Region (TMR): 86 101 * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator 87 102 * to allocate the memory, which will return aligned memory for the specified ··· 1088 1073 wrmsrq(MSR_VM_HSAVE_PA, 0); 1089 1074 } 1090 1075 1076 1076 + /* Hypervisor Fixed pages API interface */ 1077 1077 + static void snp_hv_fixed_pages_state_update(struct sev_device *sev, 1078 1078 + enum snp_hv_fixed_pages_state page_state) 1079 1079 + { 1080 1080 + struct snp_hv_fixed_pages_entry *entry; 1081 1081 + 1082 1082 + /* List is protected by sev_cmd_mutex */ 1083 1083 + lockdep_assert_held(&sev_cmd_mutex); 1084 1084 + 1085 1085 + if (list_empty(&snp_hv_fixed_pages)) 1086 1086 + return; 1087 1087 + 1088 1088 + list_for_each_entry(entry, &snp_hv_fixed_pages, list) 1089 1089 + entry->page_state = page_state; 1090 1090 + } 1091 1091 + 1092 1092 + /* 1093 1093 + * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole 1094 1094 + * 2MB pages are marked as HV_FIXED. 1095 1095 + */ 1096 1096 + struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) 1097 1097 + { 1098 1098 + struct psp_device *psp_master = psp_get_master_device(); 1099 1099 + struct snp_hv_fixed_pages_entry *entry; 1100 1100 + struct sev_device *sev; 1101 1101 + unsigned int order; 1102 1102 + struct page *page; 1103 1103 + 1104 1104 + if (!psp_master || !psp_master->sev_data) 1105 1105 + return NULL; 1106 1106 + 1107 1107 + sev = psp_master->sev_data; 1108 1108 + 1109 1109 + order = get_order(PMD_SIZE * num_2mb_pages); 1110 1110 + 1111 1111 + /* 1112 1112 + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list 1113 1113 + * also needs to be protected using the same mutex. 1114 1114 + */ 1115 1115 + guard(mutex)(&sev_cmd_mutex); 1116 1116 + 1117 1117 + /* 1118 1118 + * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed 1119 1119 + * page state, fail if SNP is already initialized. 1120 1120 + */ 1121 1121 + if (sev->snp_initialized) 1122 1122 + return NULL; 1123 1123 + 1124 1124 + /* Re-use freed pages that match the request */ 1125 1125 + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { 1126 1126 + /* Hypervisor fixed page allocator implements exact fit policy */ 1127 1127 + if (entry->order == order && entry->free) { 1128 1128 + entry->free = false; 1129 1129 + memset(page_address(entry->page), 0, 1130 1130 + (1 << entry->order) * PAGE_SIZE); 1131 1131 + return entry->page; 1132 1132 + } 1133 1133 + } 1134 1134 + 1135 1135 + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1136 1136 + if (!page) 1137 1137 + return NULL; 1138 1138 + 1139 1139 + entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1140 1140 + if (!entry) { 1141 1141 + __free_pages(page, order); 1142 1142 + return NULL; 1143 1143 + } 1144 1144 + 1145 1145 + entry->page = page; 1146 1146 + entry->order = order; 1147 1147 + list_add_tail(&entry->list, &snp_hv_fixed_pages); 1148 1148 + 1149 1149 + return page; 1150 1150 + } 1151 1151 + 1152 1152 + void snp_free_hv_fixed_pages(struct page *page) 1153 1153 + { 1154 1154 + struct psp_device *psp_master = psp_get_master_device(); 1155 1155 + struct snp_hv_fixed_pages_entry *entry, *nentry; 1156 1156 + 1157 1157 + if (!psp_master || !psp_master->sev_data) 1158 1158 + return; 1159 1159 + 1160 1160 + /* 1161 1161 + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list 1162 1162 + * also needs to be protected using the same mutex. 1163 1163 + */ 1164 1164 + guard(mutex)(&sev_cmd_mutex); 1165 1165 + 1166 1166 + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { 1167 1167 + if (entry->page != page) 1168 1168 + continue; 1169 1169 + 1170 1170 + /* 1171 1171 + * HV_FIXED page state cannot be changed until reboot 1172 1172 + * and they cannot be used by an SNP guest, so they cannot 1173 1173 + * be returned back to the page allocator. 1174 1174 + * Mark the pages as free internally to allow possible re-use. 1175 1175 + */ 1176 1176 + if (entry->page_state == HV_FIXED) { 1177 1177 + entry->free = true; 1178 1178 + } else { 1179 1179 + __free_pages(page, entry->order); 1180 1180 + list_del(&entry->list); 1181 1181 + kfree(entry); 1182 1182 + } 1183 1183 + return; 1184 1184 + } 1185 1185 + } 1186 1186 + 1187 1187 + static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list) 1188 1188 + { 1189 1189 + struct snp_hv_fixed_pages_entry *entry; 1190 1190 + struct sev_data_range *range; 1191 1191 + int num_elements; 1192 1192 + 1193 1193 + lockdep_assert_held(&sev_cmd_mutex); 1194 1194 + 1195 1195 + if (list_empty(&snp_hv_fixed_pages)) 1196 1196 + return; 1197 1197 + 1198 1198 + num_elements = list_count_nodes(&snp_hv_fixed_pages) + 1199 1199 + range_list->num_elements; 1200 1200 + 1201 1201 + /* 1202 1202 + * Ensure the list of HV_FIXED pages that will be passed to firmware 1203 1203 + * do not exceed the page-sized argument buffer. 1204 1204 + */ 1205 1205 + if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) { 1206 1206 + dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n"); 1207 1207 + return; 1208 1208 + } 1209 1209 + 1210 1210 + range = &range_list->ranges[range_list->num_elements]; 1211 1211 + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { 1212 1212 + range->base = page_to_pfn(entry->page) << PAGE_SHIFT; 1213 1213 + range->page_count = 1 << entry->order; 1214 1214 + range++; 1215 1215 + } 1216 1216 + range_list->num_elements = num_elements; 1217 1217 + } 1218 1218 + 1219 1219 + static void snp_leak_hv_fixed_pages(void) 1220 1220 + { 1221 1221 + struct snp_hv_fixed_pages_entry *entry; 1222 1222 + 1223 1223 + /* List is protected by sev_cmd_mutex */ 1224 1224 + lockdep_assert_held(&sev_cmd_mutex); 1225 1225 + 1226 1226 + if (list_empty(&snp_hv_fixed_pages)) 1227 1227 + return; 1228 1228 + 1229 1229 + list_for_each_entry(entry, &snp_hv_fixed_pages, list) 1230 1230 + if (entry->page_state == HV_FIXED) 1231 1231 + __snp_leak_pages(page_to_pfn(entry->page), 1232 1232 + 1 << entry->order, false); 1233 1233 + } 1234 1234 + 1091 1235 static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) 1092 1236 { 1093 1237 struct sev_data_range_list *range_list = arg; ··· 1337 1163 return rc; 1338 1164 } 1339 1165 1166 1166 + /* 1167 1167 + * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the 1168 1168 + * HV_Fixed page list. 1169 1169 + */ 1170 1170 + snp_add_hv_fixed_pages(sev, snp_range_list); 1171 1171 + 1340 1172 memset(&data, 0, sizeof(data)); 1341 1173 data.init_rmp = 1; 1342 1174 data.list_paddr_en = 1; ··· 1382 1202 return rc; 1383 1203 } 1384 1204 1205 1205 + snp_hv_fixed_pages_state_update(sev, HV_FIXED); 1385 1206 sev->snp_initialized = true; 1386 1207 dev_dbg(sev->dev, "SEV-SNP firmware initialized\n"); 1387 1208 ··· 1965 1784 return ret; 1966 1785 } 1967 1786 1787 1787 + snp_leak_hv_fixed_pages(); 1968 1788 sev->snp_initialized = false; 1969 1789 dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); 1970 1790

+3

drivers/crypto/ccp/sev-dev.h

reviewed

··· 65 65 void sev_pci_init(void); 66 66 void sev_pci_exit(void); 67 67 68 68 + struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages); 69 69 + void snp_free_hv_fixed_pages(struct page *page); 70 70 + 68 71 #endif /* __SEV_DEV_H */

+311

drivers/crypto/ccp/sfs.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * AMD Secure Processor Seamless Firmware Servicing support. 4 4 + * 5 5 + * Copyright (C) 2025 Advanced Micro Devices, Inc. 6 6 + * 7 7 + * Author: Ashish Kalra <ashish.kalra@amd.com> 8 8 + */ 9 9 + 10 10 + #include <linux/firmware.h> 11 11 + 12 12 + #include "sfs.h" 13 13 + #include "sev-dev.h" 14 14 + 15 15 + #define SFS_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) 16 16 + #define SFS_MAX_PAYLOAD_SIZE (2 * 1024 * 1024) 17 17 + #define SFS_NUM_2MB_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PMD_SIZE) 18 18 + #define SFS_NUM_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PAGE_SIZE) 19 19 + 20 20 + static DEFINE_MUTEX(sfs_ioctl_mutex); 21 21 + 22 22 + static struct sfs_misc_dev *misc_dev; 23 23 + 24 24 + static int send_sfs_cmd(struct sfs_device *sfs_dev, int msg) 25 25 + { 26 26 + int ret; 27 27 + 28 28 + sfs_dev->command_buf->hdr.status = 0; 29 29 + sfs_dev->command_buf->hdr.sub_cmd_id = msg; 30 30 + 31 31 + ret = psp_extended_mailbox_cmd(sfs_dev->psp, 32 32 + SFS_DEFAULT_TIMEOUT, 33 33 + (struct psp_ext_request *)sfs_dev->command_buf); 34 34 + if (ret == -EIO) { 35 35 + dev_dbg(sfs_dev->dev, 36 36 + "msg 0x%x failed with PSP error: 0x%x, extended status: 0x%x\n", 37 37 + msg, sfs_dev->command_buf->hdr.status, 38 38 + *(u32 *)sfs_dev->command_buf->buf); 39 39 + } 40 40 + 41 41 + return ret; 42 42 + } 43 43 + 44 44 + static int send_sfs_get_fw_versions(struct sfs_device *sfs_dev) 45 45 + { 46 46 + /* 47 47 + * SFS_GET_FW_VERSIONS command needs the output buffer to be 48 48 + * initialized to 0xC7 in every byte. 49 49 + */ 50 50 + memset(sfs_dev->command_buf->sfs_buffer, 0xc7, PAGE_SIZE); 51 51 + sfs_dev->command_buf->hdr.payload_size = 2 * PAGE_SIZE; 52 52 + 53 53 + return send_sfs_cmd(sfs_dev, PSP_SFS_GET_FW_VERSIONS); 54 54 + } 55 55 + 56 56 + static int send_sfs_update_package(struct sfs_device *sfs_dev, const char *payload_name) 57 57 + { 58 58 + char payload_path[PAYLOAD_NAME_SIZE + sizeof("amd/")]; 59 59 + const struct firmware *firmware; 60 60 + unsigned long package_size; 61 61 + int ret; 62 62 + 63 63 + /* Sanitize userspace provided payload name */ 64 64 + if (!strnchr(payload_name, PAYLOAD_NAME_SIZE, '\0')) 65 65 + return -EINVAL; 66 66 + 67 67 + snprintf(payload_path, sizeof(payload_path), "amd/%s", payload_name); 68 68 + 69 69 + ret = firmware_request_nowarn(&firmware, payload_path, sfs_dev->dev); 70 70 + if (ret < 0) { 71 71 + dev_warn_ratelimited(sfs_dev->dev, "firmware request failed for %s (%d)\n", 72 72 + payload_path, ret); 73 73 + return -ENOENT; 74 74 + } 75 75 + 76 76 + /* 77 77 + * SFS Update Package command's input buffer contains TEE_EXT_CMD_BUFFER 78 78 + * followed by the Update Package and it should be 64KB aligned. 79 79 + */ 80 80 + package_size = ALIGN(firmware->size + PAGE_SIZE, 0x10000U); 81 81 + 82 82 + /* 83 83 + * SFS command buffer is a pre-allocated 2MB buffer, fail update package 84 84 + * if SFS payload is larger than the pre-allocated command buffer. 85 85 + */ 86 86 + if (package_size > SFS_MAX_PAYLOAD_SIZE) { 87 87 + dev_warn_ratelimited(sfs_dev->dev, 88 88 + "SFS payload size %ld larger than maximum supported payload size of %u\n", 89 89 + package_size, SFS_MAX_PAYLOAD_SIZE); 90 90 + release_firmware(firmware); 91 91 + return -E2BIG; 92 92 + } 93 93 + 94 94 + /* 95 95 + * Copy firmware data to a HV_Fixed memory region. 96 96 + */ 97 97 + memcpy(sfs_dev->command_buf->sfs_buffer, firmware->data, firmware->size); 98 98 + sfs_dev->command_buf->hdr.payload_size = package_size; 99 99 + 100 100 + release_firmware(firmware); 101 101 + 102 102 + return send_sfs_cmd(sfs_dev, PSP_SFS_UPDATE); 103 103 + } 104 104 + 105 105 + static long sfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 106 106 + { 107 107 + struct sfs_user_get_fw_versions __user *sfs_get_fw_versions; 108 108 + struct sfs_user_update_package __user *sfs_update_package; 109 109 + struct psp_device *psp_master = psp_get_master_device(); 110 110 + char payload_name[PAYLOAD_NAME_SIZE]; 111 111 + struct sfs_device *sfs_dev; 112 112 + int ret = 0; 113 113 + 114 114 + if (!psp_master || !psp_master->sfs_data) 115 115 + return -ENODEV; 116 116 + 117 117 + sfs_dev = psp_master->sfs_data; 118 118 + 119 119 + guard(mutex)(&sfs_ioctl_mutex); 120 120 + 121 121 + switch (cmd) { 122 122 + case SFSIOCFWVERS: 123 123 + dev_dbg(sfs_dev->dev, "in SFSIOCFWVERS\n"); 124 124 + 125 125 + sfs_get_fw_versions = (struct sfs_user_get_fw_versions __user *)arg; 126 126 + 127 127 + ret = send_sfs_get_fw_versions(sfs_dev); 128 128 + if (ret && ret != -EIO) 129 129 + return ret; 130 130 + 131 131 + /* 132 132 + * Return SFS status and extended status back to userspace 133 133 + * if PSP status indicated success or command error. 134 134 + */ 135 135 + if (copy_to_user(&sfs_get_fw_versions->blob, sfs_dev->command_buf->sfs_buffer, 136 136 + PAGE_SIZE)) 137 137 + return -EFAULT; 138 138 + if (copy_to_user(&sfs_get_fw_versions->sfs_status, 139 139 + &sfs_dev->command_buf->hdr.status, 140 140 + sizeof(sfs_get_fw_versions->sfs_status))) 141 141 + return -EFAULT; 142 142 + if (copy_to_user(&sfs_get_fw_versions->sfs_extended_status, 143 143 + &sfs_dev->command_buf->buf, 144 144 + sizeof(sfs_get_fw_versions->sfs_extended_status))) 145 145 + return -EFAULT; 146 146 + break; 147 147 + case SFSIOCUPDATEPKG: 148 148 + dev_dbg(sfs_dev->dev, "in SFSIOCUPDATEPKG\n"); 149 149 + 150 150 + sfs_update_package = (struct sfs_user_update_package __user *)arg; 151 151 + 152 152 + if (copy_from_user(payload_name, sfs_update_package->payload_name, 153 153 + PAYLOAD_NAME_SIZE)) 154 154 + return -EFAULT; 155 155 + 156 156 + ret = send_sfs_update_package(sfs_dev, payload_name); 157 157 + if (ret && ret != -EIO) 158 158 + return ret; 159 159 + 160 160 + /* 161 161 + * Return SFS status and extended status back to userspace 162 162 + * if PSP status indicated success or command error. 163 163 + */ 164 164 + if (copy_to_user(&sfs_update_package->sfs_status, 165 165 + &sfs_dev->command_buf->hdr.status, 166 166 + sizeof(sfs_update_package->sfs_status))) 167 167 + return -EFAULT; 168 168 + if (copy_to_user(&sfs_update_package->sfs_extended_status, 169 169 + &sfs_dev->command_buf->buf, 170 170 + sizeof(sfs_update_package->sfs_extended_status))) 171 171 + return -EFAULT; 172 172 + break; 173 173 + default: 174 174 + ret = -EINVAL; 175 175 + } 176 176 + 177 177 + return ret; 178 178 + } 179 179 + 180 180 + static const struct file_operations sfs_fops = { 181 181 + .owner = THIS_MODULE, 182 182 + .unlocked_ioctl = sfs_ioctl, 183 183 + }; 184 184 + 185 185 + static void sfs_exit(struct kref *ref) 186 186 + { 187 187 + misc_deregister(&misc_dev->misc); 188 188 + kfree(misc_dev); 189 189 + misc_dev = NULL; 190 190 + } 191 191 + 192 192 + void sfs_dev_destroy(struct psp_device *psp) 193 193 + { 194 194 + struct sfs_device *sfs_dev = psp->sfs_data; 195 195 + 196 196 + if (!sfs_dev) 197 197 + return; 198 198 + 199 199 + /* 200 200 + * Change SFS command buffer back to the default "Write-Back" type. 201 201 + */ 202 202 + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); 203 203 + 204 204 + snp_free_hv_fixed_pages(sfs_dev->page); 205 205 + 206 206 + if (sfs_dev->misc) 207 207 + kref_put(&misc_dev->refcount, sfs_exit); 208 208 + 209 209 + psp->sfs_data = NULL; 210 210 + } 211 211 + 212 212 + /* Based on sev_misc_init() */ 213 213 + static int sfs_misc_init(struct sfs_device *sfs) 214 214 + { 215 215 + struct device *dev = sfs->dev; 216 216 + int ret; 217 217 + 218 218 + /* 219 219 + * SFS feature support can be detected on multiple devices but the SFS 220 220 + * FW commands must be issued on the master. During probe, we do not 221 221 + * know the master hence we create /dev/sfs on the first device probe. 222 222 + */ 223 223 + if (!misc_dev) { 224 224 + struct miscdevice *misc; 225 225 + 226 226 + misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL); 227 227 + if (!misc_dev) 228 228 + return -ENOMEM; 229 229 + 230 230 + misc = &misc_dev->misc; 231 231 + misc->minor = MISC_DYNAMIC_MINOR; 232 232 + misc->name = "sfs"; 233 233 + misc->fops = &sfs_fops; 234 234 + misc->mode = 0600; 235 235 + 236 236 + ret = misc_register(misc); 237 237 + if (ret) 238 238 + return ret; 239 239 + 240 240 + kref_init(&misc_dev->refcount); 241 241 + } else { 242 242 + kref_get(&misc_dev->refcount); 243 243 + } 244 244 + 245 245 + sfs->misc = misc_dev; 246 246 + dev_dbg(dev, "registered SFS device\n"); 247 247 + 248 248 + return 0; 249 249 + } 250 250 + 251 251 + int sfs_dev_init(struct psp_device *psp) 252 252 + { 253 253 + struct device *dev = psp->dev; 254 254 + struct sfs_device *sfs_dev; 255 255 + struct page *page; 256 256 + int ret = -ENOMEM; 257 257 + 258 258 + sfs_dev = devm_kzalloc(dev, sizeof(*sfs_dev), GFP_KERNEL); 259 259 + if (!sfs_dev) 260 260 + return -ENOMEM; 261 261 + 262 262 + /* 263 263 + * Pre-allocate 2MB command buffer for all SFS commands using 264 264 + * SNP HV_Fixed page allocator which also transitions the 265 265 + * SFS command buffer to HV_Fixed page state if SNP is enabled. 266 266 + */ 267 267 + page = snp_alloc_hv_fixed_pages(SFS_NUM_2MB_PAGES_CMDBUF); 268 268 + if (!page) { 269 269 + dev_dbg(dev, "Command Buffer HV-Fixed page allocation failed\n"); 270 270 + goto cleanup_dev; 271 271 + } 272 272 + sfs_dev->page = page; 273 273 + sfs_dev->command_buf = page_address(page); 274 274 + 275 275 + dev_dbg(dev, "Command buffer 0x%px to be marked as HV_Fixed\n", sfs_dev->command_buf); 276 276 + 277 277 + /* 278 278 + * SFS command buffer must be mapped as non-cacheable. 279 279 + */ 280 280 + ret = set_memory_uc((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); 281 281 + if (ret) { 282 282 + dev_dbg(dev, "Set memory uc failed\n"); 283 283 + goto cleanup_cmd_buf; 284 284 + } 285 285 + 286 286 + dev_dbg(dev, "Command buffer 0x%px marked uncacheable\n", sfs_dev->command_buf); 287 287 + 288 288 + psp->sfs_data = sfs_dev; 289 289 + sfs_dev->dev = dev; 290 290 + sfs_dev->psp = psp; 291 291 + 292 292 + ret = sfs_misc_init(sfs_dev); 293 293 + if (ret) 294 294 + goto cleanup_mem_attr; 295 295 + 296 296 + dev_notice(sfs_dev->dev, "SFS support is available\n"); 297 297 + 298 298 + return 0; 299 299 + 300 300 + cleanup_mem_attr: 301 301 + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); 302 302 + 303 303 + cleanup_cmd_buf: 304 304 + snp_free_hv_fixed_pages(page); 305 305 + 306 306 + cleanup_dev: 307 307 + psp->sfs_data = NULL; 308 308 + devm_kfree(dev, sfs_dev); 309 309 + 310 310 + return ret; 311 311 + }

+47

drivers/crypto/ccp/sfs.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 + /* 3 3 + * AMD Platform Security Processor (PSP) Seamless Firmware (SFS) Support. 4 4 + * 5 5 + * Copyright (C) 2025 Advanced Micro Devices, Inc. 6 6 + * 7 7 + * Author: Ashish Kalra <ashish.kalra@amd.com> 8 8 + */ 9 9 + 10 10 + #ifndef __SFS_H__ 11 11 + #define __SFS_H__ 12 12 + 13 13 + #include <uapi/linux/psp-sfs.h> 14 14 + 15 15 + #include <linux/device.h> 16 16 + #include <linux/miscdevice.h> 17 17 + #include <linux/psp-sev.h> 18 18 + #include <linux/psp-platform-access.h> 19 19 + #include <linux/set_memory.h> 20 20 + 21 21 + #include "psp-dev.h" 22 22 + 23 23 + struct sfs_misc_dev { 24 24 + struct kref refcount; 25 25 + struct miscdevice misc; 26 26 + }; 27 27 + 28 28 + struct sfs_command { 29 29 + struct psp_ext_req_buffer_hdr hdr; 30 30 + u8 buf[PAGE_SIZE - sizeof(struct psp_ext_req_buffer_hdr)]; 31 31 + u8 sfs_buffer[]; 32 32 + } __packed; 33 33 + 34 34 + struct sfs_device { 35 35 + struct device *dev; 36 36 + struct psp_device *psp; 37 37 + 38 38 + struct page *page; 39 39 + struct sfs_command *command_buf; 40 40 + 41 41 + struct sfs_misc_dev *misc; 42 42 + }; 43 43 + 44 44 + void sfs_dev_destroy(struct psp_device *psp); 45 45 + int sfs_dev_init(struct psp_device *psp); 46 46 + 47 47 + #endif /* __SFS_H__ */

+3 -1

drivers/firmware/efi/libstub/x86-stub.c

reviewed

··· 788 788 789 789 *kernel_entry = addr + entry; 790 790 791 791 - return efi_adjust_memory_range_protection(addr, kernel_text_size); 791 791 + return efi_adjust_memory_range_protection(addr, kernel_text_size) ?: 792 792 + efi_adjust_memory_range_protection(addr + kernel_inittext_offset, 793 793 + kernel_inittext_size); 792 794 } 793 795 794 796 static void __noreturn enter_kernel(unsigned long kernel_addr,

+8

include/linux/cc_platform.h

reviewed

··· 96 96 * enabled to run SEV-SNP guests. 97 97 */ 98 98 CC_ATTR_HOST_SEV_SNP, 99 99 + 100 100 + /** 101 101 + * @CC_ATTR_SNP_SECURE_AVIC: Secure AVIC mode is active. 102 102 + * 103 103 + * The host kernel is running with the necessary features enabled 104 104 + * to run SEV-SNP guests with full Secure AVIC capabilities. 105 105 + */ 106 106 + CC_ATTR_SNP_SECURE_AVIC, 99 107 }; 100 108 101 109 #ifdef CONFIG_ARCH_HAS_CC_PLATFORM

+2

include/linux/psp-platform-access.h

reviewed

··· 7 7 8 8 enum psp_platform_access_msg { 9 9 PSP_CMD_NONE = 0x0, 10 10 + PSP_SFS_GET_FW_VERSIONS, 11 11 + PSP_SFS_UPDATE, 10 12 PSP_CMD_HSTI_QUERY = 0x14, 11 13 PSP_I2C_REQ_BUS_CMD = 0x64, 12 14 PSP_DYNAMIC_BOOST_GET_NONCE,

+87

include/uapi/linux/psp-sfs.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ 2 2 + /* 3 3 + * Userspace interface for AMD Seamless Firmware Servicing (SFS) 4 4 + * 5 5 + * Copyright (C) 2025 Advanced Micro Devices, Inc. 6 6 + * 7 7 + * Author: Ashish Kalra <ashish.kalra@amd.com> 8 8 + */ 9 9 + 10 10 + #ifndef __PSP_SFS_USER_H__ 11 11 + #define __PSP_SFS_USER_H__ 12 12 + 13 13 + #include <linux/types.h> 14 14 + 15 15 + /** 16 16 + * SFS: AMD Seamless Firmware Support (SFS) interface 17 17 + */ 18 18 + 19 19 + #define PAYLOAD_NAME_SIZE 64 20 20 + #define TEE_EXT_CMD_BUFFER_SIZE 4096 21 21 + 22 22 + /** 23 23 + * struct sfs_user_get_fw_versions - get current level of base firmware (output). 24 24 + * @blob: current level of base firmware for ASP and patch levels (input/output). 25 25 + * @sfs_status: 32-bit SFS status value (output). 26 26 + * @sfs_extended_status: 32-bit SFS extended status value (output). 27 27 + */ 28 28 + struct sfs_user_get_fw_versions { 29 29 + __u8 blob[TEE_EXT_CMD_BUFFER_SIZE]; 30 30 + __u32 sfs_status; 31 31 + __u32 sfs_extended_status; 32 32 + } __packed; 33 33 + 34 34 + /** 35 35 + * struct sfs_user_update_package - update SFS package (input). 36 36 + * @payload_name: name of SFS package to load, verify and execute (input). 37 37 + * @sfs_status: 32-bit SFS status value (output). 38 38 + * @sfs_extended_status: 32-bit SFS extended status value (output). 39 39 + */ 40 40 + struct sfs_user_update_package { 41 41 + char payload_name[PAYLOAD_NAME_SIZE]; 42 42 + __u32 sfs_status; 43 43 + __u32 sfs_extended_status; 44 44 + } __packed; 45 45 + 46 46 + /** 47 47 + * Seamless Firmware Support (SFS) IOC 48 48 + * 49 49 + * possible return codes for all SFS IOCTLs: 50 50 + * 0: success 51 51 + * -EINVAL: invalid input 52 52 + * -E2BIG: excess data passed 53 53 + * -EFAULT: failed to copy to/from userspace 54 54 + * -EBUSY: mailbox in recovery or in use 55 55 + * -ENODEV: driver not bound with PSP device 56 56 + * -EACCES: request isn't authorized 57 57 + * -EINVAL: invalid parameter 58 58 + * -ETIMEDOUT: request timed out 59 59 + * -EAGAIN: invalid request for state machine 60 60 + * -ENOENT: not implemented 61 61 + * -ENFILE: overflow 62 62 + * -EPERM: invalid signature 63 63 + * -EIO: PSP I/O error 64 64 + */ 65 65 + #define SFS_IOC_TYPE 'S' 66 66 + 67 67 + /** 68 68 + * SFSIOCFWVERS - returns blob containing FW versions 69 69 + * ASP provides the current level of Base Firmware for the ASP 70 70 + * and the other microprocessors as well as current patch 71 71 + * level(s). 72 72 + */ 73 73 + #define SFSIOCFWVERS _IOWR(SFS_IOC_TYPE, 0x1, struct sfs_user_get_fw_versions) 74 74 + 75 75 + /** 76 76 + * SFSIOCUPDATEPKG - updates package/payload 77 77 + * ASP loads, verifies and executes the SFS package. 78 78 + * By default, the SFS package/payload is loaded from 79 79 + * /lib/firmware/amd, but alternative firmware loading 80 80 + * path can be specified using kernel parameter 81 81 + * firmware_class.path or the firmware loading path 82 82 + * can be customized using sysfs file: 83 83 + * /sys/module/firmware_class/parameters/path. 84 84 + */ 85 85 + #define SFSIOCUPDATEPKG _IOWR(SFS_IOC_TYPE, 0x2, struct sfs_user_update_package) 86 86 + 87 87 + #endif /* __PSP_SFS_USER_H__ */

+12

tools/objtool/arch/x86/decode.c

reviewed

··· 880 880 return 8; 881 881 } 882 882 } 883 883 + 884 884 + bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 885 885 + { 886 886 + switch (reloc_type(reloc)) { 887 887 + case R_X86_64_32: 888 888 + case R_X86_64_32S: 889 889 + case R_X86_64_64: 890 890 + return true; 891 891 + default: 892 892 + return false; 893 893 + } 894 894 + }

+2

tools/objtool/builtin-check.c

reviewed

··· 87 87 OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"), 88 88 OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"), 89 89 OPT_BOOLEAN(0 , "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"), 90 90 + OPT_BOOLEAN(0 , "noabs", &opts.noabs, "reject absolute references in allocatable sections"), 90 91 OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump), 91 92 92 93 OPT_GROUP("Options:"), ··· 163 162 opts.hack_noinstr || 164 163 opts.ibt || 165 164 opts.mcount || 165 165 + opts.noabs || 166 166 opts.noinstr || 167 167 opts.orc || 168 168 opts.retpoline ||

+47 -1

tools/objtool/check.c

reviewed

··· 3564 3564 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3565 3565 /* Ignore KCFI type preambles, which always fall through */ 3566 3566 if (!strncmp(func->name, "__cfi_", 6) || 3567 3567 - !strncmp(func->name, "__pfx_", 6)) 3567 3567 + !strncmp(func->name, "__pfx_", 6) || 3568 3568 + !strncmp(func->name, "__pi___cfi_", 11) || 3569 3569 + !strncmp(func->name, "__pi___pfx_", 11)) 3568 3570 return 0; 3569 3571 3570 3572 if (file->ignore_unreachables) ··· 4646 4644 disas_funcs(funcs); 4647 4645 } 4648 4646 4647 4647 + __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4648 4648 + { 4649 4649 + unsigned int type = reloc_type(reloc); 4650 4650 + size_t sz = elf_addr_size(elf); 4651 4651 + 4652 4652 + return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4653 4653 + } 4654 4654 + 4655 4655 + static int check_abs_references(struct objtool_file *file) 4656 4656 + { 4657 4657 + struct section *sec; 4658 4658 + struct reloc *reloc; 4659 4659 + int ret = 0; 4660 4660 + 4661 4661 + for_each_sec(file, sec) { 4662 4662 + /* absolute references in non-loadable sections are fine */ 4663 4663 + if (!(sec->sh.sh_flags & SHF_ALLOC)) 4664 4664 + continue; 4665 4665 + 4666 4666 + /* section must have an associated .rela section */ 4667 4667 + if (!sec->rsec) 4668 4668 + continue; 4669 4669 + 4670 4670 + /* 4671 4671 + * Special case for compiler generated metadata that is not 4672 4672 + * consumed until after boot. 4673 4673 + */ 4674 4674 + if (!strcmp(sec->name, "__patchable_function_entries")) 4675 4675 + continue; 4676 4676 + 4677 4677 + for_each_reloc(sec->rsec, reloc) { 4678 4678 + if (arch_absolute_reloc(file->elf, reloc)) { 4679 4679 + WARN("section %s has absolute relocation at offset 0x%lx", 4680 4680 + sec->name, reloc_offset(reloc)); 4681 4681 + ret++; 4682 4682 + } 4683 4683 + } 4684 4684 + } 4685 4685 + return ret; 4686 4686 + } 4687 4687 + 4649 4688 struct insn_chunk { 4650 4689 void *addr; 4651 4690 struct insn_chunk *next; ··· 4819 4776 if (ret) 4820 4777 goto out; 4821 4778 } 4779 4779 + 4780 4780 + if (opts.noabs) 4781 4781 + warnings += check_abs_references(file); 4822 4782 4823 4783 if (opts.orc && nr_insns) { 4824 4784 ret = orc_create(file);

+1

tools/objtool/include/objtool/arch.h

reviewed

··· 97 97 int arch_rewrite_retpolines(struct objtool_file *file); 98 98 99 99 bool arch_pc_relative_reloc(struct reloc *reloc); 100 100 + bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc); 100 101 101 102 unsigned int arch_reloc_size(struct reloc *reloc); 102 103 unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table);

+1

tools/objtool/include/objtool/builtin.h

reviewed

··· 26 26 bool uaccess; 27 27 int prefix; 28 28 bool cfi; 29 29 + bool noabs; 29 30 30 31 /* options: */ 31 32 bool backtrace;

-1

tools/objtool/noreturns.h

reviewed

··· 45 45 NORETURN(rust_begin_unwind) 46 46 NORETURN(rust_helper_BUG) 47 47 NORETURN(sev_es_terminate) 48 48 - NORETURN(snp_abort) 49 48 NORETURN(start_kernel) 50 49 NORETURN(stop_this_cpu) 51 50 NORETURN(usercopy_abort)