Merge tag 'x86_sev_for_v7.1_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+3 -3

arch/x86/boot/compressed/sev.c

··· 198 198 #endif 199 199 200 200 /* 201 - * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented 201 + * SNP_FEATURES_IMPL is the mask of SNP features that are implemented 202 202 * by the guest kernel. As and when a new feature is implemented in the 203 203 * guest kernel, a corresponding bit should be added to the mask. 204 204 */ 205 - #define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \ 205 + #define SNP_FEATURES_IMPL (MSR_AMD64_SNP_DEBUG_SWAP | \ 206 206 MSR_AMD64_SNP_SECURE_TSC | \ 207 207 SNP_FEATURE_SECURE_AVIC) 208 208 ··· 211 211 if (!(status & MSR_AMD64_SEV_SNP_ENABLED)) 212 212 return 0; 213 213 214 - return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT; 214 + return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_IMPL; 215 215 } 216 216 217 217 void snp_check_features(void)

+4

arch/x86/include/asm/sev.h

··· 661 661 { 662 662 __snp_leak_pages(pfn, pages, true); 663 663 } 664 + void snp_prepare(void); 665 + void snp_shutdown(void); 664 666 #else 665 667 static inline bool snp_probe_rmptable_info(void) { return false; } 666 668 static inline int snp_rmptable_init(void) { return -ENOSYS; } ··· 679 677 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} 680 678 static inline void kdump_sev_callback(void) { } 681 679 static inline void snp_fixup_e820_tables(void) {} 680 + static inline void snp_prepare(void) {} 681 + static inline void snp_shutdown(void) {} 682 682 #endif 683 683 684 684 #endif

+98 -83

arch/x86/virt/svm/sev.c

··· 117 117 118 118 static u64 rmp_cfg; 119 119 120 + static void *rmp_bookkeeping __ro_after_init; 121 + 120 122 /* Mask to apply to a PFN to get the first PFN of a 2MB page */ 121 123 #define PFN_PMD_MASK GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT) 122 124 ··· 132 130 #undef pr_fmt 133 131 #define pr_fmt(fmt) "SEV-SNP: " fmt 134 132 135 - static int __mfd_enable(unsigned int cpu) 133 + static void mfd_reconfigure(void *arg) 134 + { 135 + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 136 + return; 137 + 138 + if (arg) 139 + msr_set_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); 140 + else 141 + msr_clear_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); 142 + } 143 + 144 + static void snp_enable(void *arg) 136 145 { 137 146 u64 val; 138 147 139 148 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 140 - return 0; 141 - 142 - rdmsrq(MSR_AMD64_SYSCFG, val); 143 - 144 - val |= MSR_AMD64_SYSCFG_MFDM; 145 - 146 - wrmsrq(MSR_AMD64_SYSCFG, val); 147 - 148 - return 0; 149 - } 150 - 151 - static __init void mfd_enable(void *arg) 152 - { 153 - __mfd_enable(smp_processor_id()); 154 - } 155 - 156 - static int __snp_enable(unsigned int cpu) 157 - { 158 - u64 val; 159 - 160 - if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 161 - return 0; 149 + return; 162 150 163 151 rdmsrq(MSR_AMD64_SYSCFG, val); 164 152 ··· 156 164 val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; 157 165 158 166 wrmsrq(MSR_AMD64_SYSCFG, val); 159 - 160 - return 0; 161 - } 162 - 163 - static __init void snp_enable(void *arg) 164 - { 165 - __snp_enable(smp_processor_id()); 166 167 } 167 168 168 169 static void __init __snp_fixup_e820_tables(u64 pa) ··· 245 260 } 246 261 } 247 262 248 - static bool __init clear_rmptable_bookkeeping(void) 263 + static void clear_rmp(void) 249 264 { 250 - void *bk; 265 + unsigned int i; 266 + u64 val; 251 267 252 - bk = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB); 253 - if (!bk) { 254 - pr_err("Failed to map RMP bookkeeping area\n"); 255 - return false; 268 + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 269 + return; 270 + 271 + /* Clearing the RMP while SNP is enabled will cause an exception */ 272 + rdmsrq(MSR_AMD64_SYSCFG, val); 273 + if (WARN_ON_ONCE(val & MSR_AMD64_SYSCFG_SNP_EN)) 274 + return; 275 + 276 + memset(rmp_bookkeeping, 0, RMPTABLE_CPU_BOOKKEEPING_SZ); 277 + 278 + for (i = 0; i < rst_max_index; i++) { 279 + struct rmp_segment_desc *desc; 280 + 281 + desc = rmp_segment_table[i]; 282 + if (!desc) 283 + continue; 284 + 285 + memset(desc->rmp_entry, 0, desc->size); 256 286 } 257 - 258 - memset(bk, 0, RMPTABLE_CPU_BOOKKEEPING_SZ); 259 - 260 - memunmap(bk); 261 - 262 - return true; 263 287 } 264 288 265 289 static bool __init alloc_rmp_segment_desc(u64 segment_pa, u64 segment_size, u64 pa) ··· 488 494 static bool __init setup_rmptable(void) 489 495 { 490 496 if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) { 491 - return setup_segmented_rmptable(); 497 + if (!setup_segmented_rmptable()) 498 + return false; 492 499 } else { 493 - return setup_contiguous_rmptable(); 500 + if (!setup_contiguous_rmptable()) 501 + return false; 494 502 } 503 + 504 + rmp_bookkeeping = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB); 505 + if (!rmp_bookkeeping) { 506 + pr_err("Failed to map RMP bookkeeping area\n"); 507 + free_rmp_segment_table(); 508 + 509 + return false; 510 + } 511 + 512 + return true; 495 513 } 514 + 515 + static void clear_hsave_pa(void *arg) 516 + { 517 + wrmsrq(MSR_VM_HSAVE_PA, 0); 518 + } 519 + 520 + void snp_prepare(void) 521 + { 522 + u64 val; 523 + 524 + /* 525 + * Check if SEV-SNP is already enabled, this can happen in case of 526 + * kexec boot. 527 + */ 528 + rdmsrq(MSR_AMD64_SYSCFG, val); 529 + if (val & MSR_AMD64_SYSCFG_SNP_EN) 530 + return; 531 + 532 + clear_rmp(); 533 + 534 + cpus_read_lock(); 535 + 536 + /* 537 + * MtrrFixDramModEn is not shared between threads on a core, 538 + * therefore it must be set on all CPUs prior to enabling SNP. 539 + */ 540 + on_each_cpu(mfd_reconfigure, (void *)1, 1); 541 + on_each_cpu(snp_enable, NULL, 1); 542 + 543 + /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ 544 + on_each_cpu(clear_hsave_pa, NULL, 1); 545 + 546 + cpus_read_unlock(); 547 + } 548 + EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); 549 + 550 + void snp_shutdown(void) 551 + { 552 + u64 syscfg; 553 + 554 + rdmsrq(MSR_AMD64_SYSCFG, syscfg); 555 + if (syscfg & MSR_AMD64_SYSCFG_SNP_EN) 556 + return; 557 + 558 + clear_rmp(); 559 + on_each_cpu(mfd_reconfigure, NULL, 1); 560 + } 561 + EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp"); 496 562 497 563 /* 498 564 * Do the necessary preparations which are verified by the firmware as ··· 561 507 */ 562 508 int __init snp_rmptable_init(void) 563 509 { 564 - unsigned int i; 565 - u64 val; 566 - 567 510 if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP))) 568 511 return -ENOSYS; 569 512 ··· 569 518 570 519 if (!setup_rmptable()) 571 520 return -ENOSYS; 572 - 573 - /* 574 - * Check if SEV-SNP is already enabled, this can happen in case of 575 - * kexec boot. 576 - */ 577 - rdmsrq(MSR_AMD64_SYSCFG, val); 578 - if (val & MSR_AMD64_SYSCFG_SNP_EN) 579 - goto skip_enable; 580 - 581 - /* Zero out the RMP bookkeeping area */ 582 - if (!clear_rmptable_bookkeeping()) { 583 - free_rmp_segment_table(); 584 - return -ENOSYS; 585 - } 586 - 587 - /* Zero out the RMP entries */ 588 - for (i = 0; i < rst_max_index; i++) { 589 - struct rmp_segment_desc *desc; 590 - 591 - desc = rmp_segment_table[i]; 592 - if (!desc) 593 - continue; 594 - 595 - memset(desc->rmp_entry, 0, desc->size); 596 - } 597 - 598 - /* Flush the caches to ensure that data is written before SNP is enabled. */ 599 - wbinvd_on_all_cpus(); 600 - 601 - /* MtrrFixDramModEn must be enabled on all the CPUs prior to enabling SNP. */ 602 - on_each_cpu(mfd_enable, NULL, 1); 603 - 604 - on_each_cpu(snp_enable, NULL, 1); 605 - 606 - skip_enable: 607 - cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/rmptable_init:online", __snp_enable, NULL); 608 521 609 522 /* 610 523 * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic

+35 -27

drivers/crypto/ccp/sev-dev.c

··· 1076 1076 return __sev_init_locked(psp_ret); 1077 1077 } 1078 1078 1079 - static void snp_set_hsave_pa(void *arg) 1080 - { 1081 - wrmsrq(MSR_VM_HSAVE_PA, 0); 1082 - } 1083 - 1084 1079 /* Hypervisor Fixed pages API interface */ 1085 1080 static void snp_hv_fixed_pages_state_update(struct sev_device *sev, 1086 1081 enum snp_hv_fixed_pages_state page_state) ··· 1219 1224 1220 1225 static void snp_leak_hv_fixed_pages(void) 1221 1226 { 1222 - struct snp_hv_fixed_pages_entry *entry; 1227 + struct snp_hv_fixed_pages_entry *entry, *nentry; 1223 1228 1224 1229 /* List is protected by sev_cmd_mutex */ 1225 1230 lockdep_assert_held(&sev_cmd_mutex); ··· 1227 1232 if (list_empty(&snp_hv_fixed_pages)) 1228 1233 return; 1229 1234 1230 - list_for_each_entry(entry, &snp_hv_fixed_pages, list) 1231 - if (entry->page_state == HV_FIXED) 1235 + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { 1236 + if (entry->free && entry->page_state != HV_FIXED) 1237 + __free_pages(entry->page, entry->order); 1238 + else 1232 1239 __snp_leak_pages(page_to_pfn(entry->page), 1233 1240 1 << entry->order, false); 1241 + 1242 + list_del(&entry->list); 1243 + kfree(entry); 1244 + } 1234 1245 } 1235 1246 1236 1247 bool sev_is_snp_ciphertext_hiding_supported(void) ··· 1374 1373 return -EOPNOTSUPP; 1375 1374 } 1376 1375 1377 - /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ 1378 - on_each_cpu(snp_set_hsave_pa, NULL, 1); 1376 + snp_prepare(); 1379 1377 1380 1378 /* 1381 1379 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list ··· 2045 2045 memset(&data, 0, sizeof(data)); 2046 2046 data.len = sizeof(data); 2047 2047 data.iommu_snp_shutdown = 1; 2048 + if (sev->snp_feat_info_0.ecx & SNP_X86_SHUTDOWN_SUPPORTED) 2049 + data.x86_snp_shutdown = 1; 2048 2050 2049 2051 /* 2050 2052 * If invoked during panic handling, local interrupts are disabled ··· 2080 2078 return ret; 2081 2079 } 2082 2080 2083 - /* 2084 - * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP 2085 - * enforcement by the IOMMU and also transitions all pages 2086 - * associated with the IOMMU to the Reclaim state. 2087 - * Firmware was transitioning the IOMMU pages to Hypervisor state 2088 - * before version 1.53. But, accounting for the number of assigned 2089 - * 4kB pages in a 2M page was done incorrectly by not transitioning 2090 - * to the Reclaim state. This resulted in RMP #PF when later accessing 2091 - * the 2M page containing those pages during kexec boot. Hence, the 2092 - * firmware now transitions these pages to Reclaim state and hypervisor 2093 - * needs to transition these pages to shared state. SNP Firmware 2094 - * version 1.53 and above are needed for kexec boot. 2095 - */ 2096 - ret = amd_iommu_snp_disable(); 2097 - if (ret) { 2098 - dev_err(sev->dev, "SNP IOMMU shutdown failed\n"); 2099 - return ret; 2081 + if (data.x86_snp_shutdown) { 2082 + if (!panic) 2083 + snp_shutdown(); 2084 + snp_hv_fixed_pages_state_update(sev, ALLOCATED); 2085 + } else { 2086 + /* 2087 + * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP 2088 + * enforcement by the IOMMU and also transitions all pages 2089 + * associated with the IOMMU to the Reclaim state. 2090 + * Firmware was transitioning the IOMMU pages to Hypervisor state 2091 + * before version 1.53. But, accounting for the number of assigned 2092 + * 4kB pages in a 2M page was done incorrectly by not transitioning 2093 + * to the Reclaim state. This resulted in RMP #PF when later accessing 2094 + * the 2M page containing those pages during kexec boot. Hence, the 2095 + * firmware now transitions these pages to Reclaim state and hypervisor 2096 + * needs to transition these pages to shared state. SNP Firmware 2097 + * version 1.53 and above are needed for kexec boot. 2098 + */ 2099 + ret = amd_iommu_snp_disable(); 2100 + if (ret) { 2101 + dev_err(sev->dev, "SNP IOMMU shutdown failed\n"); 2102 + return ret; 2103 + } 2100 2104 } 2101 2105 2102 2106 snp_leak_hv_fixed_pages();

+4 -1

include/linux/psp-sev.h

··· 829 829 * 830 830 * @len: length of the command buffer read by the PSP 831 831 * @iommu_snp_shutdown: Disable enforcement of SNP in the IOMMU 832 + * @x86_snp_shutdown: Disable SNP on all cores 832 833 * @rsvd1: reserved 833 834 */ 834 835 struct sev_data_snp_shutdown_ex { 835 836 u32 len; 836 837 u32 iommu_snp_shutdown:1; 837 - u32 rsvd1:31; 838 + u32 x86_snp_shutdown:1; 839 + u32 rsvd1:30; 838 840 } __packed; 839 841 840 842 /** ··· 893 891 } __packed; 894 892 895 893 /* Feature bits in ECX */ 894 + #define SNP_X86_SHUTDOWN_SUPPORTED BIT(1) 896 895 #define SNP_RAPL_DISABLE_SUPPORTED BIT(2) 897 896 #define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3) 898 897 #define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4)

Configure Feed

Configure Feed