Merge tag 'powerpc-5.4-2' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

+3 -2

arch/powerpc/include/asm/cputable.h

··· 209 209 #define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000) 210 210 #define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000) 211 211 #define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) 212 - #define CPU_FTR_P9_TLBIE_BUG LONG_ASM_CONST(0x0000400000000000) 212 + #define CPU_FTR_P9_TLBIE_STQ_BUG LONG_ASM_CONST(0x0000400000000000) 213 213 #define CPU_FTR_P9_TIDR LONG_ASM_CONST(0x0000800000000000) 214 + #define CPU_FTR_P9_TLBIE_ERAT_BUG LONG_ASM_CONST(0x0001000000000000) 214 215 215 216 #ifndef __ASSEMBLY__ 216 217 ··· 458 457 CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ 459 458 CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ 460 459 CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \ 461 - CPU_FTR_P9_TLBIE_BUG | CPU_FTR_P9_TIDR) 460 + CPU_FTR_P9_TLBIE_STQ_BUG | CPU_FTR_P9_TLBIE_ERAT_BUG | CPU_FTR_P9_TIDR) 462 461 #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 463 462 #define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1) 464 463 #define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \

+97 -3

arch/powerpc/include/asm/kvm_ppc.h

··· 452 452 return xirr; 453 453 } 454 454 455 - static inline void kvmppc_set_host_ipi(int cpu, u8 host_ipi) 455 + /* 456 + * To avoid the need to unnecessarily exit fully to the host kernel, an IPI to 457 + * a CPU thread that's running/napping inside of a guest is by default regarded 458 + * as a request to wake the CPU (if needed) and continue execution within the 459 + * guest, potentially to process new state like externally-generated 460 + * interrupts or IPIs sent from within the guest itself (e.g. H_PROD/H_IPI). 461 + * 462 + * To force an exit to the host kernel, kvmppc_set_host_ipi() must be called 463 + * prior to issuing the IPI to set the corresponding 'host_ipi' flag in the 464 + * target CPU's PACA. To avoid unnecessary exits to the host, this flag should 465 + * be immediately cleared via kvmppc_clear_host_ipi() by the IPI handler on 466 + * the receiving side prior to processing the IPI work. 467 + * 468 + * NOTE: 469 + * 470 + * We currently issue an smp_mb() at the beginning of kvmppc_set_host_ipi(). 471 + * This is to guard against sequences such as the following: 472 + * 473 + * CPU 474 + * X: smp_muxed_ipi_set_message(): 475 + * X: smp_mb() 476 + * X: message[RESCHEDULE] = 1 477 + * X: doorbell_global_ipi(42): 478 + * X: kvmppc_set_host_ipi(42) 479 + * X: ppc_msgsnd_sync()/smp_mb() 480 + * X: ppc_msgsnd() -> 42 481 + * 42: doorbell_exception(): // from CPU X 482 + * 42: ppc_msgsync() 483 + * 105: smp_muxed_ipi_set_message(): 484 + * 105: smb_mb() 485 + * // STORE DEFERRED DUE TO RE-ORDERING 486 + * --105: message[CALL_FUNCTION] = 1 487 + * | 105: doorbell_global_ipi(42): 488 + * | 105: kvmppc_set_host_ipi(42) 489 + * | 42: kvmppc_clear_host_ipi(42) 490 + * | 42: smp_ipi_demux_relaxed() 491 + * | 42: // returns to executing guest 492 + * | // RE-ORDERED STORE COMPLETES 493 + * ->105: message[CALL_FUNCTION] = 1 494 + * 105: ppc_msgsnd_sync()/smp_mb() 495 + * 105: ppc_msgsnd() -> 42 496 + * 42: local_paca->kvm_hstate.host_ipi == 0 // IPI ignored 497 + * 105: // hangs waiting on 42 to process messages/call_single_queue 498 + * 499 + * We also issue an smp_mb() at the end of kvmppc_clear_host_ipi(). This is 500 + * to guard against sequences such as the following (as well as to create 501 + * a read-side pairing with the barrier in kvmppc_set_host_ipi()): 502 + * 503 + * CPU 504 + * X: smp_muxed_ipi_set_message(): 505 + * X: smp_mb() 506 + * X: message[RESCHEDULE] = 1 507 + * X: doorbell_global_ipi(42): 508 + * X: kvmppc_set_host_ipi(42) 509 + * X: ppc_msgsnd_sync()/smp_mb() 510 + * X: ppc_msgsnd() -> 42 511 + * 42: doorbell_exception(): // from CPU X 512 + * 42: ppc_msgsync() 513 + * // STORE DEFERRED DUE TO RE-ORDERING 514 + * -- 42: kvmppc_clear_host_ipi(42) 515 + * | 42: smp_ipi_demux_relaxed() 516 + * | 105: smp_muxed_ipi_set_message(): 517 + * | 105: smb_mb() 518 + * | 105: message[CALL_FUNCTION] = 1 519 + * | 105: doorbell_global_ipi(42): 520 + * | 105: kvmppc_set_host_ipi(42) 521 + * | // RE-ORDERED STORE COMPLETES 522 + * -> 42: kvmppc_clear_host_ipi(42) 523 + * 42: // returns to executing guest 524 + * 105: ppc_msgsnd_sync()/smp_mb() 525 + * 105: ppc_msgsnd() -> 42 526 + * 42: local_paca->kvm_hstate.host_ipi == 0 // IPI ignored 527 + * 105: // hangs waiting on 42 to process messages/call_single_queue 528 + */ 529 + static inline void kvmppc_set_host_ipi(int cpu) 456 530 { 457 - paca_ptrs[cpu]->kvm_hstate.host_ipi = host_ipi; 531 + /* 532 + * order stores of IPI messages vs. setting of host_ipi flag 533 + * 534 + * pairs with the barrier in kvmppc_clear_host_ipi() 535 + */ 536 + smp_mb(); 537 + paca_ptrs[cpu]->kvm_hstate.host_ipi = 1; 538 + } 539 + 540 + static inline void kvmppc_clear_host_ipi(int cpu) 541 + { 542 + paca_ptrs[cpu]->kvm_hstate.host_ipi = 0; 543 + /* 544 + * order clearing of host_ipi flag vs. processing of IPI messages 545 + * 546 + * pairs with the barrier in kvmppc_set_host_ipi() 547 + */ 548 + smp_mb(); 458 549 } 459 550 460 551 static inline void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu) ··· 577 486 return 0; 578 487 } 579 488 580 - static inline void kvmppc_set_host_ipi(int cpu, u8 host_ipi) 489 + static inline void kvmppc_set_host_ipi(int cpu) 490 + {} 491 + 492 + static inline void kvmppc_clear_host_ipi(int cpu) 581 493 {} 582 494 583 495 static inline void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu)

+6 -3

arch/powerpc/include/asm/reg.h

··· 475 475 #define HMER_DEBUG_TRIG (1ul << (63 - 17)) /* Debug trigger */ 476 476 #define SPRN_HMEER 0x151 /* Hyp maintenance exception enable reg */ 477 477 #define SPRN_PCR 0x152 /* Processor compatibility register */ 478 - #define PCR_VEC_DIS (1ul << (63-0)) /* Vec. disable (bit NA since POWER8) */ 479 - #define PCR_VSX_DIS (1ul << (63-1)) /* VSX disable (bit NA since POWER8) */ 480 - #define PCR_TM_DIS (1ul << (63-2)) /* Trans. memory disable (POWER8) */ 478 + #define PCR_VEC_DIS (__MASK(63-0)) /* Vec. disable (bit NA since POWER8) */ 479 + #define PCR_VSX_DIS (__MASK(63-1)) /* VSX disable (bit NA since POWER8) */ 480 + #define PCR_TM_DIS (__MASK(63-2)) /* Trans. memory disable (POWER8) */ 481 + #define PCR_HIGH_BITS (PCR_VEC_DIS | PCR_VSX_DIS | PCR_TM_DIS) 481 482 /* 482 483 * These bits are used in the function kvmppc_set_arch_compat() to specify and 483 484 * determine both the compatibility level which we want to emulate and the ··· 487 486 #define PCR_ARCH_207 0x8 /* Architecture 2.07 */ 488 487 #define PCR_ARCH_206 0x4 /* Architecture 2.06 */ 489 488 #define PCR_ARCH_205 0x2 /* Architecture 2.05 */ 489 + #define PCR_LOW_BITS (PCR_ARCH_207 | PCR_ARCH_206 | PCR_ARCH_205) 490 + #define PCR_MASK ~(PCR_HIGH_BITS | PCR_LOW_BITS) /* PCR Reserved Bits */ 490 491 #define SPRN_HEIR 0x153 /* Hypervisor Emulated Instruction Register */ 491 492 #define SPRN_TLBINDEXR 0x154 /* P7 TLB control register */ 492 493 #define SPRN_TLBVPNR 0x155 /* P7 TLB control register */

+6

arch/powerpc/kernel/cpu_setup_power.S

··· 23 23 beqlr 24 24 li r0,0 25 25 mtspr SPRN_LPID,r0 26 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 26 27 mtspr SPRN_PCR,r0 27 28 mfspr r3,SPRN_LPCR 28 29 li r4,(LPCR_LPES1 >> LPCR_LPES_SH) ··· 38 37 beqlr 39 38 li r0,0 40 39 mtspr SPRN_LPID,r0 40 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 41 41 mtspr SPRN_PCR,r0 42 42 mfspr r3,SPRN_LPCR 43 43 li r4,(LPCR_LPES1 >> LPCR_LPES_SH) ··· 56 54 beqlr 57 55 li r0,0 58 56 mtspr SPRN_LPID,r0 57 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 59 58 mtspr SPRN_PCR,r0 60 59 mfspr r3,SPRN_LPCR 61 60 ori r3, r3, LPCR_PECEDH ··· 79 76 beqlr 80 77 li r0,0 81 78 mtspr SPRN_LPID,r0 79 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 82 80 mtspr SPRN_PCR,r0 83 81 mfspr r3,SPRN_LPCR 84 82 ori r3, r3, LPCR_PECEDH ··· 102 98 mtspr SPRN_PSSCR,r0 103 99 mtspr SPRN_LPID,r0 104 100 mtspr SPRN_PID,r0 101 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 105 102 mtspr SPRN_PCR,r0 106 103 mfspr r3,SPRN_LPCR 107 104 LOAD_REG_IMMEDIATE(r4, LPCR_PECEDH | LPCR_PECE_HVEE | LPCR_HVICE | LPCR_HEIC) ··· 128 123 mtspr SPRN_PSSCR,r0 129 124 mtspr SPRN_LPID,r0 130 125 mtspr SPRN_PID,r0 126 + LOAD_REG_IMMEDIATE(r0, PCR_MASK) 131 127 mtspr SPRN_PCR,r0 132 128 mfspr r3,SPRN_LPCR 133 129 LOAD_REG_IMMEDIATE(r4, LPCR_PECEDH | LPCR_PECE_HVEE | LPCR_HVICE | LPCR_HEIC)

+3 -3

arch/powerpc/kernel/dbell.c

··· 33 33 { 34 34 u32 tag = get_hard_smp_processor_id(cpu); 35 35 36 - kvmppc_set_host_ipi(cpu, 1); 36 + kvmppc_set_host_ipi(cpu); 37 37 /* Order previous accesses vs. msgsnd, which is treated as a store */ 38 38 ppc_msgsnd_sync(); 39 39 ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); ··· 48 48 { 49 49 u32 tag = cpu_thread_in_core(cpu); 50 50 51 - kvmppc_set_host_ipi(cpu, 1); 51 + kvmppc_set_host_ipi(cpu); 52 52 /* Order previous accesses vs. msgsnd, which is treated as a store */ 53 53 ppc_msgsnd_sync(); 54 54 ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); ··· 84 84 85 85 may_hard_irq_enable(); 86 86 87 - kvmppc_set_host_ipi(smp_processor_id(), 0); 87 + kvmppc_clear_host_ipi(smp_processor_id()); 88 88 __this_cpu_inc(irq_stat.doorbell_irqs); 89 89 90 90 smp_ipi_demux_relaxed(); /* already performed the barrier */

+32 -3

arch/powerpc/kernel/dt_cpu_ftrs.c

··· 101 101 if (hv_mode) { 102 102 mtspr(SPRN_LPID, 0); 103 103 mtspr(SPRN_HFSCR, system_registers.hfscr); 104 - mtspr(SPRN_PCR, 0); 104 + mtspr(SPRN_PCR, PCR_MASK); 105 105 } 106 106 mtspr(SPRN_FSCR, system_registers.fscr); 107 107 ··· 144 144 mtspr(SPRN_HFSCR, 0); 145 145 } 146 146 mtspr(SPRN_FSCR, 0); 147 + mtspr(SPRN_PCR, PCR_MASK); 147 148 148 149 /* 149 150 * LPCR does not get cleared, to match behaviour with secondaries ··· 692 691 return true; 693 692 } 694 693 694 + /* 695 + * Handle POWER9 broadcast tlbie invalidation issue using 696 + * cpu feature flag. 697 + */ 698 + static __init void update_tlbie_feature_flag(unsigned long pvr) 699 + { 700 + if (PVR_VER(pvr) == PVR_POWER9) { 701 + /* 702 + * Set the tlbie feature flag for anything below 703 + * Nimbus DD 2.3 and Cumulus DD 1.3 704 + */ 705 + if ((pvr & 0xe000) == 0) { 706 + /* Nimbus */ 707 + if ((pvr & 0xfff) < 0x203) 708 + cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 709 + } else if ((pvr & 0xc000) == 0) { 710 + /* Cumulus */ 711 + if ((pvr & 0xfff) < 0x103) 712 + cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 713 + } else { 714 + WARN_ONCE(1, "Unknown PVR"); 715 + cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; 716 + } 717 + 718 + cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; 719 + } 720 + } 721 + 695 722 static __init void cpufeatures_cpu_quirks(void) 696 723 { 697 - int version = mfspr(SPRN_PVR); 724 + unsigned long version = mfspr(SPRN_PVR); 698 725 699 726 /* 700 727 * Not all quirks can be derived from the cpufeatures device tree. ··· 741 712 742 713 if ((version & 0xffff0000) == 0x004e0000) { 743 714 cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); 744 - cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG; 745 715 cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; 746 716 } 747 717 718 + update_tlbie_feature_flag(version); 748 719 /* 749 720 * PKEY was not in the initial base or feature node 750 721 * specification, but it should become optional in the next

+2 -2

arch/powerpc/kernel/eeh.c

··· 1960 1960 pci_err(pdev, "Going to break: %pR\n", bar); 1961 1961 1962 1962 if (pdev->is_virtfn) { 1963 - #ifndef CONFIG_IOV 1963 + #ifndef CONFIG_PCI_IOV 1964 1964 return -ENXIO; 1965 1965 #else 1966 1966 /* ··· 1980 1980 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 1981 1981 pos += PCI_SRIOV_CTRL; 1982 1982 bit = PCI_SRIOV_CTRL_MSE; 1983 - #endif /* !CONFIG_IOV */ 1983 + #endif /* !CONFIG_PCI_IOV */ 1984 1984 } else { 1985 1985 bit = PCI_COMMAND_MEMORY; 1986 1986 pos = PCI_COMMAND;

+7 -4

arch/powerpc/kvm/book3s_hv.c

··· 401 401 402 402 spin_lock(&vc->lock); 403 403 vc->arch_compat = arch_compat; 404 - /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */ 405 - vc->pcr = host_pcr_bit - guest_pcr_bit; 404 + /* 405 + * Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit 406 + * Also set all reserved PCR bits 407 + */ 408 + vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK; 406 409 spin_unlock(&vc->lock); 407 410 408 411 return 0; ··· 3413 3410 } 3414 3411 3415 3412 if (vc->pcr) 3416 - mtspr(SPRN_PCR, vc->pcr); 3413 + mtspr(SPRN_PCR, vc->pcr | PCR_MASK); 3417 3414 mtspr(SPRN_DPDES, vc->dpdes); 3418 3415 mtspr(SPRN_VTB, vc->vtb); 3419 3416 ··· 3493 3490 vc->vtb = mfspr(SPRN_VTB); 3494 3491 mtspr(SPRN_DPDES, 0); 3495 3492 if (vc->pcr) 3496 - mtspr(SPRN_PCR, 0); 3493 + mtspr(SPRN_PCR, PCR_MASK); 3497 3494 3498 3495 if (vc->tb_offset_applied) { 3499 3496 u64 new_tb = mftb() - vc->tb_offset_applied;

+3 -3

arch/powerpc/kvm/book3s_hv_nested.c

··· 29 29 { 30 30 struct kvmppc_vcore *vc = vcpu->arch.vcore; 31 31 32 - hr->pcr = vc->pcr; 32 + hr->pcr = vc->pcr | PCR_MASK; 33 33 hr->dpdes = vc->dpdes; 34 34 hr->hfscr = vcpu->arch.hfscr; 35 35 hr->tb_offset = vc->tb_offset; ··· 65 65 hr->lpid = swab32(hr->lpid); 66 66 hr->vcpu_token = swab32(hr->vcpu_token); 67 67 hr->lpcr = swab64(hr->lpcr); 68 - hr->pcr = swab64(hr->pcr); 68 + hr->pcr = swab64(hr->pcr) | PCR_MASK; 69 69 hr->amor = swab64(hr->amor); 70 70 hr->dpdes = swab64(hr->dpdes); 71 71 hr->hfscr = swab64(hr->hfscr); ··· 148 148 { 149 149 struct kvmppc_vcore *vc = vcpu->arch.vcore; 150 150 151 - vc->pcr = hr->pcr; 151 + vc->pcr = hr->pcr | PCR_MASK; 152 152 vc->dpdes = hr->dpdes; 153 153 vcpu->arch.hfscr = hr->hfscr; 154 154 vcpu->arch.dawr = hr->dawr0;

+32 -10

arch/powerpc/kvm/book3s_hv_rm_mmu.c

··· 433 433 (HPTE_R_KEY_HI | HPTE_R_KEY_LO)); 434 434 } 435 435 436 + static inline void fixup_tlbie_lpid(unsigned long rb_value, unsigned long lpid) 437 + { 438 + 439 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 440 + /* Radix flush for a hash guest */ 441 + 442 + unsigned long rb,rs,prs,r,ric; 443 + 444 + rb = PPC_BIT(52); /* IS = 2 */ 445 + rs = 0; /* lpid = 0 */ 446 + prs = 0; /* partition scoped */ 447 + r = 1; /* radix format */ 448 + ric = 0; /* RIC_FLSUH_TLB */ 449 + 450 + /* 451 + * Need the extra ptesync to make sure we don't 452 + * re-order the tlbie 453 + */ 454 + asm volatile("ptesync": : :"memory"); 455 + asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 456 + : : "r"(rb), "i"(r), "i"(prs), 457 + "i"(ric), "r"(rs) : "memory"); 458 + } 459 + 460 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 461 + asm volatile("ptesync": : :"memory"); 462 + asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : : 463 + "r" (rb_value), "r" (lpid)); 464 + } 465 + } 466 + 436 467 static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, 437 468 long npages, int global, bool need_sync) 438 469 { ··· 482 451 "r" (rbvalues[i]), "r" (kvm->arch.lpid)); 483 452 } 484 453 485 - if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 486 - /* 487 - * Need the extra ptesync to make sure we don't 488 - * re-order the tlbie 489 - */ 490 - asm volatile("ptesync": : :"memory"); 491 - asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : : 492 - "r" (rbvalues[0]), "r" (kvm->arch.lpid)); 493 - } 494 - 454 + fixup_tlbie_lpid(rbvalues[i - 1], kvm->arch.lpid); 495 455 asm volatile("eieio; tlbsync; ptesync" : : : "memory"); 496 456 } else { 497 457 if (need_sync)

+1 -1

arch/powerpc/kvm/book3s_hv_rm_xics.c

··· 58 58 hcpu = hcore << threads_shift; 59 59 kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu; 60 60 smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION); 61 - kvmppc_set_host_ipi(hcpu, 1); 61 + kvmppc_set_host_ipi(hcpu); 62 62 smp_mb(); 63 63 kvmhv_rm_send_ipi(hcpu); 64 64 }

+6 -4

arch/powerpc/kvm/book3s_hv_rmhandlers.S

··· 644 644 645 645 /* Load guest PCR value to select appropriate compat mode */ 646 646 37: ld r7, VCORE_PCR(r5) 647 - cmpdi r7, 0 647 + LOAD_REG_IMMEDIATE(r6, PCR_MASK) 648 + cmpld r7, r6 648 649 beq 38f 650 + or r7, r7, r6 649 651 mtspr SPRN_PCR, r7 650 652 38: 651 653 ··· 1915 1913 1916 1914 /* Reset PCR */ 1917 1915 ld r0, VCORE_PCR(r5) 1918 - cmpdi r0, 0 1916 + LOAD_REG_IMMEDIATE(r6, PCR_MASK) 1917 + cmpld r0, r6 1919 1918 beq 18f 1920 - li r0, 0 1921 - mtspr SPRN_PCR, r0 1919 + mtspr SPRN_PCR, r6 1922 1920 18: 1923 1921 /* Signal secondary CPUs to continue */ 1924 1922 stb r0,VCORE_IN_GUEST(r5)

+27 -4

arch/powerpc/mm/book3s64/hash_native.c

··· 197 197 return va; 198 198 } 199 199 200 - static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize) 200 + static inline void fixup_tlbie_vpn(unsigned long vpn, int psize, 201 + int apsize, int ssize) 201 202 { 202 - if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 203 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 204 + /* Radix flush for a hash guest */ 205 + 206 + unsigned long rb,rs,prs,r,ric; 207 + 208 + rb = PPC_BIT(52); /* IS = 2 */ 209 + rs = 0; /* lpid = 0 */ 210 + prs = 0; /* partition scoped */ 211 + r = 1; /* radix format */ 212 + ric = 0; /* RIC_FLSUH_TLB */ 213 + 214 + /* 215 + * Need the extra ptesync to make sure we don't 216 + * re-order the tlbie 217 + */ 218 + asm volatile("ptesync": : :"memory"); 219 + asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 220 + : : "r"(rb), "i"(r), "i"(prs), 221 + "i"(ric), "r"(rs) : "memory"); 222 + } 223 + 224 + 225 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 203 226 /* Need the extra ptesync to ensure we don't reorder tlbie*/ 204 227 asm volatile("ptesync": : :"memory"); 205 228 ___tlbie(vpn, psize, apsize, ssize); ··· 306 283 asm volatile("ptesync": : :"memory"); 307 284 } else { 308 285 __tlbie(vpn, psize, apsize, ssize); 309 - fixup_tlbie(vpn, psize, apsize, ssize); 286 + fixup_tlbie_vpn(vpn, psize, apsize, ssize); 310 287 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 311 288 } 312 289 if (lock_tlbie && !use_local) ··· 879 856 /* 880 857 * Just do one more with the last used values. 881 858 */ 882 - fixup_tlbie(vpn, psize, psize, ssize); 859 + fixup_tlbie_vpn(vpn, psize, psize, ssize); 883 860 asm volatile("eieio; tlbsync; ptesync":::"memory"); 884 861 885 862 if (lock_tlbie)

+14 -1

arch/powerpc/mm/book3s64/mmu_context.c

··· 256 256 #ifdef CONFIG_SPAPR_TCE_IOMMU 257 257 WARN_ON_ONCE(!list_empty(&mm->context.iommu_group_mem_list)); 258 258 #endif 259 + /* 260 + * For tasks which were successfully initialized we end up calling 261 + * arch_exit_mmap() which clears the process table entry. And 262 + * arch_exit_mmap() is called before the required fullmm TLB flush 263 + * which does a RIC=2 flush. Hence for an initialized task, we do clear 264 + * any cached process table entries. 265 + * 266 + * The condition below handles the error case during task init. We have 267 + * set the process table entry early and if we fail a task 268 + * initialization, we need to ensure the process table entry is zeroed. 269 + * We need not worry about process table entry caches because the task 270 + * never ran with the PID value. 271 + */ 259 272 if (radix_enabled()) 260 - WARN_ON(process_tb[mm->context.id].prtb0 != 0); 273 + process_tb[mm->context.id].prtb0 = 0; 261 274 else 262 275 subpage_prot_free(mm); 263 276 destroy_contexts(&mm->context);

+74 -10

arch/powerpc/mm/book3s64/radix_tlb.c

··· 196 196 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 197 197 } 198 198 199 - static inline void fixup_tlbie(void) 199 + 200 + static inline void fixup_tlbie_va(unsigned long va, unsigned long pid, 201 + unsigned long ap) 200 202 { 201 - unsigned long pid = 0; 203 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 204 + asm volatile("ptesync": : :"memory"); 205 + __tlbie_va(va, 0, ap, RIC_FLUSH_TLB); 206 + } 207 + 208 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 209 + asm volatile("ptesync": : :"memory"); 210 + __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 211 + } 212 + } 213 + 214 + static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid, 215 + unsigned long ap) 216 + { 217 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 218 + asm volatile("ptesync": : :"memory"); 219 + __tlbie_pid(0, RIC_FLUSH_TLB); 220 + } 221 + 222 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 223 + asm volatile("ptesync": : :"memory"); 224 + __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 225 + } 226 + } 227 + 228 + static inline void fixup_tlbie_pid(unsigned long pid) 229 + { 230 + /* 231 + * We can use any address for the invalidation, pick one which is 232 + * probably unused as an optimisation. 233 + */ 202 234 unsigned long va = ((1UL << 52) - 1); 203 235 204 - if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 236 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 237 + asm volatile("ptesync": : :"memory"); 238 + __tlbie_pid(0, RIC_FLUSH_TLB); 239 + } 240 + 241 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 205 242 asm volatile("ptesync": : :"memory"); 206 243 __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 207 244 } 208 245 } 209 246 247 + 248 + static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, 249 + unsigned long ap) 250 + { 251 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 252 + asm volatile("ptesync": : :"memory"); 253 + __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB); 254 + } 255 + 256 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 257 + asm volatile("ptesync": : :"memory"); 258 + __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB); 259 + } 260 + } 261 + 210 262 static inline void fixup_tlbie_lpid(unsigned long lpid) 211 263 { 264 + /* 265 + * We can use any address for the invalidation, pick one which is 266 + * probably unused as an optimisation. 267 + */ 212 268 unsigned long va = ((1UL << 52) - 1); 213 269 214 - if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 270 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 271 + asm volatile("ptesync": : :"memory"); 272 + __tlbie_lpid(0, RIC_FLUSH_TLB); 273 + } 274 + 275 + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 215 276 asm volatile("ptesync": : :"memory"); 216 277 __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 217 278 } ··· 319 258 switch (ric) { 320 259 case RIC_FLUSH_TLB: 321 260 __tlbie_pid(pid, RIC_FLUSH_TLB); 261 + fixup_tlbie_pid(pid); 322 262 break; 323 263 case RIC_FLUSH_PWC: 324 264 __tlbie_pid(pid, RIC_FLUSH_PWC); ··· 327 265 case RIC_FLUSH_ALL: 328 266 default: 329 267 __tlbie_pid(pid, RIC_FLUSH_ALL); 268 + fixup_tlbie_pid(pid); 330 269 } 331 - fixup_tlbie(); 332 270 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 333 271 } 334 272 ··· 377 315 switch (ric) { 378 316 case RIC_FLUSH_TLB: 379 317 __tlbie_lpid(lpid, RIC_FLUSH_TLB); 318 + fixup_tlbie_lpid(lpid); 380 319 break; 381 320 case RIC_FLUSH_PWC: 382 321 __tlbie_lpid(lpid, RIC_FLUSH_PWC); ··· 385 322 case RIC_FLUSH_ALL: 386 323 default: 387 324 __tlbie_lpid(lpid, RIC_FLUSH_ALL); 325 + fixup_tlbie_lpid(lpid); 388 326 } 389 - fixup_tlbie_lpid(lpid); 390 327 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 391 328 } 392 329 ··· 453 390 454 391 for (addr = start; addr < end; addr += page_size) 455 392 __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); 393 + 394 + fixup_tlbie_va_range(addr - page_size, pid, ap); 456 395 } 457 396 458 397 static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, ··· 464 399 465 400 asm volatile("ptesync": : :"memory"); 466 401 __tlbie_va(va, pid, ap, ric); 467 - fixup_tlbie(); 402 + fixup_tlbie_va(va, pid, ap); 468 403 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 469 404 } 470 405 ··· 522 457 523 458 asm volatile("ptesync": : :"memory"); 524 459 __tlbie_lpid_va(va, lpid, ap, ric); 525 - fixup_tlbie_lpid(lpid); 460 + fixup_tlbie_lpid_va(va, lpid, ap); 526 461 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 527 462 } 528 463 ··· 534 469 if (also_pwc) 535 470 __tlbie_pid(pid, RIC_FLUSH_PWC); 536 471 __tlbie_va_range(start, end, pid, page_size, psize); 537 - fixup_tlbie(); 538 472 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 539 473 } 540 474 ··· 920 856 if (gflush) 921 857 __tlbie_va_range(gstart, gend, pid, 922 858 PUD_SIZE, MMU_PAGE_1G); 923 - fixup_tlbie(); 859 + 924 860 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 925 861 } else { 926 862 _tlbiel_va_range_multicast(mm,

+26 -8

arch/powerpc/mm/kasan/kasan_init_32.c

··· 12 12 #include <asm/code-patching.h> 13 13 #include <mm/mmu_decl.h> 14 14 15 + static pgprot_t kasan_prot_ro(void) 16 + { 17 + if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) 18 + return PAGE_READONLY; 19 + 20 + return PAGE_KERNEL_RO; 21 + } 22 + 15 23 static void kasan_populate_pte(pte_t *ptep, pgprot_t prot) 16 24 { 17 25 unsigned long va = (unsigned long)kasan_early_shadow_page; ··· 34 26 { 35 27 pmd_t *pmd; 36 28 unsigned long k_cur, k_next; 29 + pgprot_t prot = slab_is_available() ? kasan_prot_ro() : PAGE_KERNEL; 37 30 38 31 pmd = pmd_offset(pud_offset(pgd_offset_k(k_start), k_start), k_start); 39 32 ··· 52 43 53 44 if (!new) 54 45 return -ENOMEM; 55 - if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) 56 - kasan_populate_pte(new, PAGE_READONLY); 57 - else 58 - kasan_populate_pte(new, PAGE_KERNEL_RO); 46 + kasan_populate_pte(new, prot); 59 47 60 48 smp_wmb(); /* See comment in __pte_alloc */ 61 49 ··· 109 103 110 104 static void __init kasan_remap_early_shadow_ro(void) 111 105 { 112 - if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) 113 - kasan_populate_pte(kasan_early_shadow_pte, PAGE_READONLY); 114 - else 115 - kasan_populate_pte(kasan_early_shadow_pte, PAGE_KERNEL_RO); 106 + pgprot_t prot = kasan_prot_ro(); 107 + unsigned long k_start = KASAN_SHADOW_START; 108 + unsigned long k_end = KASAN_SHADOW_END; 109 + unsigned long k_cur; 110 + phys_addr_t pa = __pa(kasan_early_shadow_page); 116 111 112 + kasan_populate_pte(kasan_early_shadow_pte, prot); 113 + 114 + for (k_cur = k_start & PAGE_MASK; k_cur < k_end; k_cur += PAGE_SIZE) { 115 + pmd_t *pmd = pmd_offset(pud_offset(pgd_offset_k(k_cur), k_cur), k_cur); 116 + pte_t *ptep = pte_offset_kernel(pmd, k_cur); 117 + 118 + if ((pte_val(*ptep) & PTE_RPN_MASK) != pa) 119 + continue; 120 + 121 + __set_pte_at(&init_mm, k_cur, ptep, pfn_pte(PHYS_PFN(pa), prot), 0); 122 + } 117 123 flush_tlb_kernel_range(KASAN_SHADOW_START, KASAN_SHADOW_END); 118 124 } 119 125

+1 -1

arch/powerpc/platforms/powernv/smp.c

··· 193 193 * for coming online, which are handled via 194 194 * generic_check_cpu_restart() calls. 195 195 */ 196 - kvmppc_set_host_ipi(cpu, 0); 196 + kvmppc_clear_host_ipi(cpu); 197 197 198 198 srr1 = pnv_cpu_offline(cpu); 199 199

+161 -2

arch/powerpc/platforms/pseries/lpar.c

··· 56 56 EXPORT_SYMBOL(plpar_hcall9); 57 57 EXPORT_SYMBOL(plpar_hcall_norets); 58 58 59 + /* 60 + * H_BLOCK_REMOVE supported block size for this page size in segment who's base 61 + * page size is that page size. 62 + * 63 + * The first index is the segment base page size, the second one is the actual 64 + * page size. 65 + */ 66 + static int hblkrm_size[MMU_PAGE_COUNT][MMU_PAGE_COUNT] __ro_after_init; 67 + 68 + /* 69 + * Due to the involved complexity, and that the current hypervisor is only 70 + * returning this value or 0, we are limiting the support of the H_BLOCK_REMOVE 71 + * buffer size to 8 size block. 72 + */ 73 + #define HBLKRM_SUPPORTED_BLOCK_SIZE 8 74 + 59 75 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 60 76 static u8 dtl_mask = DTL_LOG_PREEMPT; 61 77 #else ··· 1000 984 #define HBLKR_CTRL_ERRNOTFOUND 0x8800000000000000UL 1001 985 #define HBLKR_CTRL_ERRBUSY 0xa000000000000000UL 1002 986 987 + /* 988 + * Returned true if we are supporting this block size for the specified segment 989 + * base page size and actual page size. 990 + * 991 + * Currently, we only support 8 size block. 992 + */ 993 + static inline bool is_supported_hlbkrm(int bpsize, int psize) 994 + { 995 + return (hblkrm_size[bpsize][psize] == HBLKRM_SUPPORTED_BLOCK_SIZE); 996 + } 997 + 1003 998 /** 1004 999 * H_BLOCK_REMOVE caller. 1005 1000 * @idx should point to the latest @param entry set with a PTEX. ··· 1170 1143 if (lock_tlbie) 1171 1144 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); 1172 1145 1173 - if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE)) 1146 + /* Assuming THP size is 16M */ 1147 + if (is_supported_hlbkrm(psize, MMU_PAGE_16M)) 1174 1148 hugepage_block_invalidate(slot, vpn, count, psize, ssize); 1175 1149 else 1176 1150 hugepage_bulk_invalidate(slot, vpn, count, psize, ssize); ··· 1340 1312 } 1341 1313 1342 1314 /* 1315 + * TLB Block Invalidate Characteristics 1316 + * 1317 + * These characteristics define the size of the block the hcall H_BLOCK_REMOVE 1318 + * is able to process for each couple segment base page size, actual page size. 1319 + * 1320 + * The ibm,get-system-parameter properties is returning a buffer with the 1321 + * following layout: 1322 + * 1323 + * [ 2 bytes size of the RTAS buffer (excluding these 2 bytes) ] 1324 + * ----------------- 1325 + * TLB Block Invalidate Specifiers: 1326 + * [ 1 byte LOG base 2 of the TLB invalidate block size being specified ] 1327 + * [ 1 byte Number of page sizes (N) that are supported for the specified 1328 + * TLB invalidate block size ] 1329 + * [ 1 byte Encoded segment base page size and actual page size 1330 + * MSB=0 means 4k segment base page size and actual page size 1331 + * MSB=1 the penc value in mmu_psize_def ] 1332 + * ... 1333 + * ----------------- 1334 + * Next TLB Block Invalidate Specifiers... 1335 + * ----------------- 1336 + * [ 0 ] 1337 + */ 1338 + static inline void set_hblkrm_bloc_size(int bpsize, int psize, 1339 + unsigned int block_size) 1340 + { 1341 + if (block_size > hblkrm_size[bpsize][psize]) 1342 + hblkrm_size[bpsize][psize] = block_size; 1343 + } 1344 + 1345 + /* 1346 + * Decode the Encoded segment base page size and actual page size. 1347 + * PAPR specifies: 1348 + * - bit 7 is the L bit 1349 + * - bits 0-5 are the penc value 1350 + * If the L bit is 0, this means 4K segment base page size and actual page size 1351 + * otherwise the penc value should be read. 1352 + */ 1353 + #define HBLKRM_L_MASK 0x80 1354 + #define HBLKRM_PENC_MASK 0x3f 1355 + static inline void __init check_lp_set_hblkrm(unsigned int lp, 1356 + unsigned int block_size) 1357 + { 1358 + unsigned int bpsize, psize; 1359 + 1360 + /* First, check the L bit, if not set, this means 4K */ 1361 + if ((lp & HBLKRM_L_MASK) == 0) { 1362 + set_hblkrm_bloc_size(MMU_PAGE_4K, MMU_PAGE_4K, block_size); 1363 + return; 1364 + } 1365 + 1366 + lp &= HBLKRM_PENC_MASK; 1367 + for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++) { 1368 + struct mmu_psize_def *def = &mmu_psize_defs[bpsize]; 1369 + 1370 + for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { 1371 + if (def->penc[psize] == lp) { 1372 + set_hblkrm_bloc_size(bpsize, psize, block_size); 1373 + return; 1374 + } 1375 + } 1376 + } 1377 + } 1378 + 1379 + #define SPLPAR_TLB_BIC_TOKEN 50 1380 + 1381 + /* 1382 + * The size of the TLB Block Invalidate Characteristics is variable. But at the 1383 + * maximum it will be the number of possible page sizes *2 + 10 bytes. 1384 + * Currently MMU_PAGE_COUNT is 16, which means 42 bytes. Use a cache line size 1385 + * (128 bytes) for the buffer to get plenty of space. 1386 + */ 1387 + #define SPLPAR_TLB_BIC_MAXLENGTH 128 1388 + 1389 + void __init pseries_lpar_read_hblkrm_characteristics(void) 1390 + { 1391 + unsigned char local_buffer[SPLPAR_TLB_BIC_MAXLENGTH]; 1392 + int call_status, len, idx, bpsize; 1393 + 1394 + spin_lock(&rtas_data_buf_lock); 1395 + memset(rtas_data_buf, 0, RTAS_DATA_BUF_SIZE); 1396 + call_status = rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1, 1397 + NULL, 1398 + SPLPAR_TLB_BIC_TOKEN, 1399 + __pa(rtas_data_buf), 1400 + RTAS_DATA_BUF_SIZE); 1401 + memcpy(local_buffer, rtas_data_buf, SPLPAR_TLB_BIC_MAXLENGTH); 1402 + local_buffer[SPLPAR_TLB_BIC_MAXLENGTH - 1] = '\0'; 1403 + spin_unlock(&rtas_data_buf_lock); 1404 + 1405 + if (call_status != 0) { 1406 + pr_warn("%s %s Error calling get-system-parameter (0x%x)\n", 1407 + __FILE__, __func__, call_status); 1408 + return; 1409 + } 1410 + 1411 + /* 1412 + * The first two (2) bytes of the data in the buffer are the length of 1413 + * the returned data, not counting these first two (2) bytes. 1414 + */ 1415 + len = be16_to_cpu(*((u16 *)local_buffer)) + 2; 1416 + if (len > SPLPAR_TLB_BIC_MAXLENGTH) { 1417 + pr_warn("%s too large returned buffer %d", __func__, len); 1418 + return; 1419 + } 1420 + 1421 + idx = 2; 1422 + while (idx < len) { 1423 + u8 block_shift = local_buffer[idx++]; 1424 + u32 block_size; 1425 + unsigned int npsize; 1426 + 1427 + if (!block_shift) 1428 + break; 1429 + 1430 + block_size = 1 << block_shift; 1431 + 1432 + for (npsize = local_buffer[idx++]; 1433 + npsize > 0 && idx < len; npsize--) 1434 + check_lp_set_hblkrm((unsigned int) local_buffer[idx++], 1435 + block_size); 1436 + } 1437 + 1438 + for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++) 1439 + for (idx = 0; idx < MMU_PAGE_COUNT; idx++) 1440 + if (hblkrm_size[bpsize][idx]) 1441 + pr_info("H_BLOCK_REMOVE supports base psize:%d psize:%d block size:%d", 1442 + bpsize, idx, hblkrm_size[bpsize][idx]); 1443 + } 1444 + 1445 + /* 1343 1446 * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie 1344 1447 * lock. 1345 1448 */ ··· 1489 1330 if (lock_tlbie) 1490 1331 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); 1491 1332 1492 - if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE)) { 1333 + if (is_supported_hlbkrm(batch->psize, batch->psize)) { 1493 1334 do_block_remove(number, batch, param); 1494 1335 goto out; 1495 1336 }

+50 -22

arch/powerpc/platforms/pseries/papr_scm.c

··· 65 65 cond_resched(); 66 66 } while (rc == H_BUSY); 67 67 68 - if (rc) { 69 - /* H_OVERLAP needs a separate error path */ 70 - if (rc == H_OVERLAP) 71 - return -EBUSY; 72 - 73 - dev_err(&p->pdev->dev, "bind err: %lld\n", rc); 74 - return -ENXIO; 75 - } 68 + if (rc) 69 + return rc; 76 70 77 71 p->bound_addr = saved; 78 - 79 - dev_dbg(&p->pdev->dev, "bound drc %x to %pR\n", p->drc_index, &p->res); 80 - 81 - return 0; 72 + dev_dbg(&p->pdev->dev, "bound drc 0x%x to %pR\n", p->drc_index, &p->res); 73 + return rc; 82 74 } 83 75 84 - static int drc_pmem_unbind(struct papr_scm_priv *p) 76 + static void drc_pmem_unbind(struct papr_scm_priv *p) 85 77 { 86 78 unsigned long ret[PLPAR_HCALL_BUFSIZE]; 87 79 uint64_t token = 0; 88 80 int64_t rc; 89 81 90 - dev_dbg(&p->pdev->dev, "unbind drc %x\n", p->drc_index); 82 + dev_dbg(&p->pdev->dev, "unbind drc 0x%x\n", p->drc_index); 91 83 92 84 /* NB: unbind has the same retry requirements as drc_pmem_bind() */ 93 85 do { ··· 102 110 if (rc) 103 111 dev_err(&p->pdev->dev, "unbind error: %lld\n", rc); 104 112 else 105 - dev_dbg(&p->pdev->dev, "unbind drc %x complete\n", 113 + dev_dbg(&p->pdev->dev, "unbind drc 0x%x complete\n", 106 114 p->drc_index); 107 115 108 - return rc == H_SUCCESS ? 0 : -ENXIO; 116 + return; 109 117 } 118 + 119 + static int drc_pmem_query_n_bind(struct papr_scm_priv *p) 120 + { 121 + unsigned long start_addr; 122 + unsigned long end_addr; 123 + unsigned long ret[PLPAR_HCALL_BUFSIZE]; 124 + int64_t rc; 125 + 126 + 127 + rc = plpar_hcall(H_SCM_QUERY_BLOCK_MEM_BINDING, ret, 128 + p->drc_index, 0); 129 + if (rc) 130 + goto err_out; 131 + start_addr = ret[0]; 132 + 133 + /* Make sure the full region is bound. */ 134 + rc = plpar_hcall(H_SCM_QUERY_BLOCK_MEM_BINDING, ret, 135 + p->drc_index, p->blocks - 1); 136 + if (rc) 137 + goto err_out; 138 + end_addr = ret[0]; 139 + 140 + if ((end_addr - start_addr) != ((p->blocks - 1) * p->block_size)) 141 + goto err_out; 142 + 143 + p->bound_addr = start_addr; 144 + dev_dbg(&p->pdev->dev, "bound drc 0x%x to %pR\n", p->drc_index, &p->res); 145 + return rc; 146 + 147 + err_out: 148 + dev_info(&p->pdev->dev, 149 + "Failed to query, trying an unbind followed by bind"); 150 + drc_pmem_unbind(p); 151 + return drc_pmem_bind(p); 152 + } 153 + 110 154 111 155 static int papr_scm_meta_get(struct papr_scm_priv *p, 112 156 struct nd_cmd_get_config_data_hdr *hdr) ··· 464 436 rc = drc_pmem_bind(p); 465 437 466 438 /* If phyp says drc memory still bound then force unbound and retry */ 467 - if (rc == -EBUSY) { 468 - dev_warn(&pdev->dev, "Retrying bind after unbinding\n"); 469 - drc_pmem_unbind(p); 470 - rc = drc_pmem_bind(p); 471 - } 439 + if (rc == H_OVERLAP) 440 + rc = drc_pmem_query_n_bind(p); 472 441 473 - if (rc) 442 + if (rc != H_SUCCESS) { 443 + dev_err(&p->pdev->dev, "bind err: %d\n", rc); 444 + rc = -ENXIO; 474 445 goto err; 446 + } 475 447 476 448 /* setup the resource for the newly bound range */ 477 449 p->res.start = p->bound_addr;

+1

arch/powerpc/platforms/pseries/pseries.h

··· 112 112 int dlpar_workqueue_init(void); 113 113 114 114 void pseries_setup_rfi_flush(void); 115 + void pseries_lpar_read_hblkrm_characteristics(void); 115 116 116 117 #endif /* _PSERIES_PSERIES_H */

+1

arch/powerpc/platforms/pseries/setup.c

··· 744 744 745 745 pseries_setup_rfi_flush(); 746 746 setup_stf_barrier(); 747 + pseries_lpar_read_hblkrm_characteristics(); 747 748 748 749 /* By default, only probe PCI (can be overridden by rtas_pci) */ 749 750 pci_add_flags(PCI_PROBE_ONLY);

+3 -3

arch/powerpc/sysdev/xics/icp-native.c

··· 140 140 141 141 static void icp_native_cause_ipi(int cpu) 142 142 { 143 - kvmppc_set_host_ipi(cpu, 1); 143 + kvmppc_set_host_ipi(cpu); 144 144 icp_native_set_qirr(cpu, IPI_PRIORITY); 145 145 } 146 146 ··· 179 179 if (vec == XICS_IPI) { 180 180 /* Clear pending IPI */ 181 181 int cpu = smp_processor_id(); 182 - kvmppc_set_host_ipi(cpu, 0); 182 + kvmppc_clear_host_ipi(cpu); 183 183 icp_native_set_qirr(cpu, 0xff); 184 184 } else { 185 185 pr_err("XICS: hw interrupt 0x%x to offline cpu, disabling\n", ··· 200 200 { 201 201 int cpu = smp_processor_id(); 202 202 203 - kvmppc_set_host_ipi(cpu, 0); 203 + kvmppc_clear_host_ipi(cpu); 204 204 icp_native_set_qirr(cpu, 0xff); 205 205 206 206 return smp_ipi_demux();

+3 -3

arch/powerpc/sysdev/xics/icp-opal.c

··· 126 126 { 127 127 int hw_cpu = get_hard_smp_processor_id(cpu); 128 128 129 - kvmppc_set_host_ipi(cpu, 1); 129 + kvmppc_set_host_ipi(cpu); 130 130 opal_int_set_mfrr(hw_cpu, IPI_PRIORITY); 131 131 } 132 132 ··· 134 134 { 135 135 int cpu = smp_processor_id(); 136 136 137 - kvmppc_set_host_ipi(cpu, 0); 137 + kvmppc_clear_host_ipi(cpu); 138 138 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), 0xff); 139 139 140 140 return smp_ipi_demux(); ··· 157 157 if (vec == XICS_IPI) { 158 158 /* Clear pending IPI */ 159 159 int cpu = smp_processor_id(); 160 - kvmppc_set_host_ipi(cpu, 0); 160 + kvmppc_clear_host_ipi(cpu); 161 161 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), 0xff); 162 162 } else { 163 163 pr_err("XICS: hw interrupt 0x%x to offline cpu, "

+2

tools/testing/selftests/powerpc/mm/Makefile

··· 4 4 5 5 TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \ 6 6 large_vm_fork_separation 7 + TEST_GEN_PROGS_EXTENDED := tlbie_test 7 8 TEST_GEN_FILES := tempfile 8 9 9 10 top_srcdir = ../../../../.. ··· 20 19 $(OUTPUT)/tempfile: 21 20 dd if=/dev/zero of=$@ bs=64k count=1 22 21 22 + $(OUTPUT)/tlbie_test: LDLIBS += -lpthread

+734

tools/testing/selftests/powerpc/mm/tlbie_test.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + /* 4 + * Copyright 2019, Nick Piggin, Gautham R. Shenoy, Aneesh Kumar K.V, IBM Corp. 5 + */ 6 + 7 + /* 8 + * 9 + * Test tlbie/mtpidr race. We have 4 threads doing flush/load/compare/store 10 + * sequence in a loop. The same threads also rung a context switch task 11 + * that does sched_yield() in loop. 12 + * 13 + * The snapshot thread mark the mmap area PROT_READ in between, make a copy 14 + * and copy it back to the original area. This helps us to detect if any 15 + * store continued to happen after we marked the memory PROT_READ. 16 + */ 17 + 18 + #define _GNU_SOURCE 19 + #include <stdio.h> 20 + #include <sys/mman.h> 21 + #include <sys/types.h> 22 + #include <sys/wait.h> 23 + #include <sys/ipc.h> 24 + #include <sys/shm.h> 25 + #include <sys/stat.h> 26 + #include <sys/time.h> 27 + #include <linux/futex.h> 28 + #include <unistd.h> 29 + #include <asm/unistd.h> 30 + #include <string.h> 31 + #include <stdlib.h> 32 + #include <fcntl.h> 33 + #include <sched.h> 34 + #include <time.h> 35 + #include <stdarg.h> 36 + #include <sched.h> 37 + #include <pthread.h> 38 + #include <signal.h> 39 + #include <sys/prctl.h> 40 + 41 + static inline void dcbf(volatile unsigned int *addr) 42 + { 43 + __asm__ __volatile__ ("dcbf %y0; sync" : : "Z"(*(unsigned char *)addr) : "memory"); 44 + } 45 + 46 + static void err_msg(char *msg) 47 + { 48 + 49 + time_t now; 50 + time(&now); 51 + printf("=================================\n"); 52 + printf(" Error: %s\n", msg); 53 + printf(" %s", ctime(&now)); 54 + printf("=================================\n"); 55 + exit(1); 56 + } 57 + 58 + static char *map1; 59 + static char *map2; 60 + static pid_t rim_process_pid; 61 + 62 + /* 63 + * A "rim-sequence" is defined to be the sequence of the following 64 + * operations performed on a memory word: 65 + * 1) FLUSH the contents of that word. 66 + * 2) LOAD the contents of that word. 67 + * 3) COMPARE the contents of that word with the content that was 68 + * previously stored at that word 69 + * 4) STORE new content into that word. 70 + * 71 + * The threads in this test that perform the rim-sequence are termed 72 + * as rim_threads. 73 + */ 74 + 75 + /* 76 + * A "corruption" is defined to be the failed COMPARE operation in a 77 + * rim-sequence. 78 + * 79 + * A rim_thread that detects a corruption informs about it to all the 80 + * other rim_threads, and the mem_snapshot thread. 81 + */ 82 + static volatile unsigned int corruption_found; 83 + 84 + /* 85 + * This defines the maximum number of rim_threads in this test. 86 + * 87 + * The THREAD_ID_BITS denote the number of bits required 88 + * to represent the thread_ids [0..MAX_THREADS - 1]. 89 + * We are being a bit paranoid here and set it to 8 bits, 90 + * though 6 bits suffice. 91 + * 92 + */ 93 + #define MAX_THREADS 64 94 + #define THREAD_ID_BITS 8 95 + #define THREAD_ID_MASK ((1 << THREAD_ID_BITS) - 1) 96 + static unsigned int rim_thread_ids[MAX_THREADS]; 97 + static pthread_t rim_threads[MAX_THREADS]; 98 + 99 + 100 + /* 101 + * Each rim_thread works on an exclusive "chunk" of size 102 + * RIM_CHUNK_SIZE. 103 + * 104 + * The ith rim_thread works on the ith chunk. 105 + * 106 + * The ith chunk begins at 107 + * map1 + (i * RIM_CHUNK_SIZE) 108 + */ 109 + #define RIM_CHUNK_SIZE 1024 110 + #define BITS_PER_BYTE 8 111 + #define WORD_SIZE (sizeof(unsigned int)) 112 + #define WORD_BITS (WORD_SIZE * BITS_PER_BYTE) 113 + #define WORDS_PER_CHUNK (RIM_CHUNK_SIZE/WORD_SIZE) 114 + 115 + static inline char *compute_chunk_start_addr(unsigned int thread_id) 116 + { 117 + char *chunk_start; 118 + 119 + chunk_start = (char *)((unsigned long)map1 + 120 + (thread_id * RIM_CHUNK_SIZE)); 121 + 122 + return chunk_start; 123 + } 124 + 125 + /* 126 + * The "word-offset" of a word-aligned address inside a chunk, is 127 + * defined to be the number of words that precede the address in that 128 + * chunk. 129 + * 130 + * WORD_OFFSET_BITS denote the number of bits required to represent 131 + * the word-offsets of all the word-aligned addresses of a chunk. 132 + */ 133 + #define WORD_OFFSET_BITS (__builtin_ctz(WORDS_PER_CHUNK)) 134 + #define WORD_OFFSET_MASK ((1 << WORD_OFFSET_BITS) - 1) 135 + 136 + static inline unsigned int compute_word_offset(char *start, unsigned int *addr) 137 + { 138 + unsigned int delta_bytes, ret; 139 + delta_bytes = (unsigned long)addr - (unsigned long)start; 140 + 141 + ret = delta_bytes/WORD_SIZE; 142 + 143 + return ret; 144 + } 145 + 146 + /* 147 + * A "sweep" is defined to be the sequential execution of the 148 + * rim-sequence by a rim_thread on its chunk one word at a time, 149 + * starting from the first word of its chunk and ending with the last 150 + * word of its chunk. 151 + * 152 + * Each sweep of a rim_thread is uniquely identified by a sweep_id. 153 + * SWEEP_ID_BITS denote the number of bits required to represent 154 + * the sweep_ids of rim_threads. 155 + * 156 + * As to why SWEEP_ID_BITS are computed as a function of THREAD_ID_BITS, 157 + * WORD_OFFSET_BITS, and WORD_BITS, see the "store-pattern" below. 158 + */ 159 + #define SWEEP_ID_BITS (WORD_BITS - (THREAD_ID_BITS + WORD_OFFSET_BITS)) 160 + #define SWEEP_ID_MASK ((1 << SWEEP_ID_BITS) - 1) 161 + 162 + /* 163 + * A "store-pattern" is the word-pattern that is stored into a word 164 + * location in the 4)STORE step of the rim-sequence. 165 + * 166 + * In the store-pattern, we shall encode: 167 + * 168 + * - The thread-id of the rim_thread performing the store 169 + * (The most significant THREAD_ID_BITS) 170 + * 171 + * - The word-offset of the address into which the store is being 172 + * performed (The next WORD_OFFSET_BITS) 173 + * 174 + * - The sweep_id of the current sweep in which the store is 175 + * being performed. (The lower SWEEP_ID_BITS) 176 + * 177 + * Store Pattern: 32 bits 178 + * |------------------|--------------------|---------------------------------| 179 + * | Thread id | Word offset | sweep_id | 180 + * |------------------|--------------------|---------------------------------| 181 + * THREAD_ID_BITS WORD_OFFSET_BITS SWEEP_ID_BITS 182 + * 183 + * In the store pattern, the (Thread-id + Word-offset) uniquely identify the 184 + * address to which the store is being performed i.e, 185 + * address == map1 + 186 + * (Thread-id * RIM_CHUNK_SIZE) + (Word-offset * WORD_SIZE) 187 + * 188 + * And the sweep_id in the store pattern identifies the time when the 189 + * store was performed by the rim_thread. 190 + * 191 + * We shall use this property in the 3)COMPARE step of the 192 + * rim-sequence. 193 + */ 194 + #define SWEEP_ID_SHIFT 0 195 + #define WORD_OFFSET_SHIFT (SWEEP_ID_BITS) 196 + #define THREAD_ID_SHIFT (WORD_OFFSET_BITS + SWEEP_ID_BITS) 197 + 198 + /* 199 + * Compute the store pattern for a given thread with id @tid, at 200 + * location @addr in the sweep identified by @sweep_id 201 + */ 202 + static inline unsigned int compute_store_pattern(unsigned int tid, 203 + unsigned int *addr, 204 + unsigned int sweep_id) 205 + { 206 + unsigned int ret = 0; 207 + char *start = compute_chunk_start_addr(tid); 208 + unsigned int word_offset = compute_word_offset(start, addr); 209 + 210 + ret += (tid & THREAD_ID_MASK) << THREAD_ID_SHIFT; 211 + ret += (word_offset & WORD_OFFSET_MASK) << WORD_OFFSET_SHIFT; 212 + ret += (sweep_id & SWEEP_ID_MASK) << SWEEP_ID_SHIFT; 213 + return ret; 214 + } 215 + 216 + /* Extract the thread-id from the given store-pattern */ 217 + static inline unsigned int extract_tid(unsigned int pattern) 218 + { 219 + unsigned int ret; 220 + 221 + ret = (pattern >> THREAD_ID_SHIFT) & THREAD_ID_MASK; 222 + return ret; 223 + } 224 + 225 + /* Extract the word-offset from the given store-pattern */ 226 + static inline unsigned int extract_word_offset(unsigned int pattern) 227 + { 228 + unsigned int ret; 229 + 230 + ret = (pattern >> WORD_OFFSET_SHIFT) & WORD_OFFSET_MASK; 231 + 232 + return ret; 233 + } 234 + 235 + /* Extract the sweep-id from the given store-pattern */ 236 + static inline unsigned int extract_sweep_id(unsigned int pattern) 237 + 238 + { 239 + unsigned int ret; 240 + 241 + ret = (pattern >> SWEEP_ID_SHIFT) & SWEEP_ID_MASK; 242 + 243 + return ret; 244 + } 245 + 246 + /************************************************************ 247 + * * 248 + * Logging the output of the verification * 249 + * * 250 + ************************************************************/ 251 + #define LOGDIR_NAME_SIZE 100 252 + static char logdir[LOGDIR_NAME_SIZE]; 253 + 254 + static FILE *fp[MAX_THREADS]; 255 + static const char logfilename[] ="Thread-%02d-Chunk"; 256 + 257 + static inline void start_verification_log(unsigned int tid, 258 + unsigned int *addr, 259 + unsigned int cur_sweep_id, 260 + unsigned int prev_sweep_id) 261 + { 262 + FILE *f; 263 + char logfile[30]; 264 + char path[LOGDIR_NAME_SIZE + 30]; 265 + char separator[2] = "/"; 266 + char *chunk_start = compute_chunk_start_addr(tid); 267 + unsigned int size = RIM_CHUNK_SIZE; 268 + 269 + sprintf(logfile, logfilename, tid); 270 + strcpy(path, logdir); 271 + strcat(path, separator); 272 + strcat(path, logfile); 273 + f = fopen(path, "w"); 274 + 275 + if (!f) { 276 + err_msg("Unable to create logfile\n"); 277 + } 278 + 279 + fp[tid] = f; 280 + 281 + fprintf(f, "----------------------------------------------------------\n"); 282 + fprintf(f, "PID = %d\n", rim_process_pid); 283 + fprintf(f, "Thread id = %02d\n", tid); 284 + fprintf(f, "Chunk Start Addr = 0x%016lx\n", (unsigned long)chunk_start); 285 + fprintf(f, "Chunk Size = %d\n", size); 286 + fprintf(f, "Next Store Addr = 0x%016lx\n", (unsigned long)addr); 287 + fprintf(f, "Current sweep-id = 0x%08x\n", cur_sweep_id); 288 + fprintf(f, "Previous sweep-id = 0x%08x\n", prev_sweep_id); 289 + fprintf(f, "----------------------------------------------------------\n"); 290 + } 291 + 292 + static inline void log_anamoly(unsigned int tid, unsigned int *addr, 293 + unsigned int expected, unsigned int observed) 294 + { 295 + FILE *f = fp[tid]; 296 + 297 + fprintf(f, "Thread %02d: Addr 0x%lx: Expected 0x%x, Observed 0x%x\n", 298 + tid, (unsigned long)addr, expected, observed); 299 + fprintf(f, "Thread %02d: Expected Thread id = %02d\n", tid, extract_tid(expected)); 300 + fprintf(f, "Thread %02d: Observed Thread id = %02d\n", tid, extract_tid(observed)); 301 + fprintf(f, "Thread %02d: Expected Word offset = %03d\n", tid, extract_word_offset(expected)); 302 + fprintf(f, "Thread %02d: Observed Word offset = %03d\n", tid, extract_word_offset(observed)); 303 + fprintf(f, "Thread %02d: Expected sweep-id = 0x%x\n", tid, extract_sweep_id(expected)); 304 + fprintf(f, "Thread %02d: Observed sweep-id = 0x%x\n", tid, extract_sweep_id(observed)); 305 + fprintf(f, "----------------------------------------------------------\n"); 306 + } 307 + 308 + static inline void end_verification_log(unsigned int tid, unsigned nr_anamolies) 309 + { 310 + FILE *f = fp[tid]; 311 + char logfile[30]; 312 + char path[LOGDIR_NAME_SIZE + 30]; 313 + char separator[] = "/"; 314 + 315 + fclose(f); 316 + 317 + if (nr_anamolies == 0) { 318 + remove(path); 319 + return; 320 + } 321 + 322 + sprintf(logfile, logfilename, tid); 323 + strcpy(path, logdir); 324 + strcat(path, separator); 325 + strcat(path, logfile); 326 + 327 + printf("Thread %02d chunk has %d corrupted words. For details check %s\n", 328 + tid, nr_anamolies, path); 329 + } 330 + 331 + /* 332 + * When a COMPARE step of a rim-sequence fails, the rim_thread informs 333 + * everyone else via the shared_memory pointed to by 334 + * corruption_found variable. On seeing this, every thread verifies the 335 + * content of its chunk as follows. 336 + * 337 + * Suppose a thread identified with @tid was about to store (but not 338 + * yet stored) to @next_store_addr in its current sweep identified 339 + * @cur_sweep_id. Let @prev_sweep_id indicate the previous sweep_id. 340 + * 341 + * This implies that for all the addresses @addr < @next_store_addr, 342 + * Thread @tid has already performed a store as part of its current 343 + * sweep. Hence we expect the content of such @addr to be: 344 + * |-------------------------------------------------| 345 + * | tid | word_offset(addr) | cur_sweep_id | 346 + * |-------------------------------------------------| 347 + * 348 + * Since Thread @tid is yet to perform stores on address 349 + * @next_store_addr and above, we expect the content of such an 350 + * address @addr to be: 351 + * |-------------------------------------------------| 352 + * | tid | word_offset(addr) | prev_sweep_id | 353 + * |-------------------------------------------------| 354 + * 355 + * The verifier function @verify_chunk does this verification and logs 356 + * any anamolies that it finds. 357 + */ 358 + static void verify_chunk(unsigned int tid, unsigned int *next_store_addr, 359 + unsigned int cur_sweep_id, 360 + unsigned int prev_sweep_id) 361 + { 362 + unsigned int *iter_ptr; 363 + unsigned int size = RIM_CHUNK_SIZE; 364 + unsigned int expected; 365 + unsigned int observed; 366 + char *chunk_start = compute_chunk_start_addr(tid); 367 + 368 + int nr_anamolies = 0; 369 + 370 + start_verification_log(tid, next_store_addr, 371 + cur_sweep_id, prev_sweep_id); 372 + 373 + for (iter_ptr = (unsigned int *)chunk_start; 374 + (unsigned long)iter_ptr < (unsigned long)chunk_start + size; 375 + iter_ptr++) { 376 + unsigned int expected_sweep_id; 377 + 378 + if (iter_ptr < next_store_addr) { 379 + expected_sweep_id = cur_sweep_id; 380 + } else { 381 + expected_sweep_id = prev_sweep_id; 382 + } 383 + 384 + expected = compute_store_pattern(tid, iter_ptr, expected_sweep_id); 385 + 386 + dcbf((volatile unsigned int*)iter_ptr); //Flush before reading 387 + observed = *iter_ptr; 388 + 389 + if (observed != expected) { 390 + nr_anamolies++; 391 + log_anamoly(tid, iter_ptr, expected, observed); 392 + } 393 + } 394 + 395 + end_verification_log(tid, nr_anamolies); 396 + } 397 + 398 + static void set_pthread_cpu(pthread_t th, int cpu) 399 + { 400 + cpu_set_t run_cpu_mask; 401 + struct sched_param param; 402 + 403 + CPU_ZERO(&run_cpu_mask); 404 + CPU_SET(cpu, &run_cpu_mask); 405 + pthread_setaffinity_np(th, sizeof(cpu_set_t), &run_cpu_mask); 406 + 407 + param.sched_priority = 1; 408 + if (0 && sched_setscheduler(0, SCHED_FIFO, &param) == -1) { 409 + /* haven't reproduced with this setting, it kills random preemption which may be a factor */ 410 + fprintf(stderr, "could not set SCHED_FIFO, run as root?\n"); 411 + } 412 + } 413 + 414 + static void set_mycpu(int cpu) 415 + { 416 + cpu_set_t run_cpu_mask; 417 + struct sched_param param; 418 + 419 + CPU_ZERO(&run_cpu_mask); 420 + CPU_SET(cpu, &run_cpu_mask); 421 + sched_setaffinity(0, sizeof(cpu_set_t), &run_cpu_mask); 422 + 423 + param.sched_priority = 1; 424 + if (0 && sched_setscheduler(0, SCHED_FIFO, &param) == -1) { 425 + fprintf(stderr, "could not set SCHED_FIFO, run as root?\n"); 426 + } 427 + } 428 + 429 + static volatile int segv_wait; 430 + 431 + static void segv_handler(int signo, siginfo_t *info, void *extra) 432 + { 433 + while (segv_wait) { 434 + sched_yield(); 435 + } 436 + 437 + } 438 + 439 + static void set_segv_handler(void) 440 + { 441 + struct sigaction sa; 442 + 443 + sa.sa_flags = SA_SIGINFO; 444 + sa.sa_sigaction = segv_handler; 445 + 446 + if (sigaction(SIGSEGV, &sa, NULL) == -1) { 447 + perror("sigaction"); 448 + exit(EXIT_FAILURE); 449 + } 450 + } 451 + 452 + int timeout = 0; 453 + /* 454 + * This function is executed by every rim_thread. 455 + * 456 + * This function performs sweeps over the exclusive chunks of the 457 + * rim_threads executing the rim-sequence one word at a time. 458 + */ 459 + static void *rim_fn(void *arg) 460 + { 461 + unsigned int tid = *((unsigned int *)arg); 462 + 463 + int size = RIM_CHUNK_SIZE; 464 + char *chunk_start = compute_chunk_start_addr(tid); 465 + 466 + unsigned int prev_sweep_id; 467 + unsigned int cur_sweep_id = 0; 468 + 469 + /* word access */ 470 + unsigned int pattern = cur_sweep_id; 471 + unsigned int *pattern_ptr = &pattern; 472 + unsigned int *w_ptr, read_data; 473 + 474 + set_segv_handler(); 475 + 476 + /* 477 + * Let us initialize the chunk: 478 + * 479 + * Each word-aligned address addr in the chunk, 480 + * is initialized to : 481 + * |-------------------------------------------------| 482 + * | tid | word_offset(addr) | 0 | 483 + * |-------------------------------------------------| 484 + */ 485 + for (w_ptr = (unsigned int *)chunk_start; 486 + (unsigned long)w_ptr < (unsigned long)(chunk_start) + size; 487 + w_ptr++) { 488 + 489 + *pattern_ptr = compute_store_pattern(tid, w_ptr, cur_sweep_id); 490 + *w_ptr = *pattern_ptr; 491 + } 492 + 493 + while (!corruption_found && !timeout) { 494 + prev_sweep_id = cur_sweep_id; 495 + cur_sweep_id = cur_sweep_id + 1; 496 + 497 + for (w_ptr = (unsigned int *)chunk_start; 498 + (unsigned long)w_ptr < (unsigned long)(chunk_start) + size; 499 + w_ptr++) { 500 + unsigned int old_pattern; 501 + 502 + /* 503 + * Compute the pattern that we would have 504 + * stored at this location in the previous 505 + * sweep. 506 + */ 507 + old_pattern = compute_store_pattern(tid, w_ptr, prev_sweep_id); 508 + 509 + /* 510 + * FLUSH:Ensure that we flush the contents of 511 + * the cache before loading 512 + */ 513 + dcbf((volatile unsigned int*)w_ptr); //Flush 514 + 515 + /* LOAD: Read the value */ 516 + read_data = *w_ptr; //Load 517 + 518 + /* 519 + * COMPARE: Is it the same as what we had stored 520 + * in the previous sweep ? It better be! 521 + */ 522 + if (read_data != old_pattern) { 523 + /* No it isn't! Tell everyone */ 524 + corruption_found = 1; 525 + } 526 + 527 + /* 528 + * Before performing a store, let us check if 529 + * any rim_thread has found a corruption. 530 + */ 531 + if (corruption_found || timeout) { 532 + /* 533 + * Yes. Someone (including us!) has found 534 + * a corruption :( 535 + * 536 + * Let us verify that our chunk is 537 + * correct. 538 + */ 539 + /* But first, let us allow the dust to settle down! */ 540 + verify_chunk(tid, w_ptr, cur_sweep_id, prev_sweep_id); 541 + 542 + return 0; 543 + } 544 + 545 + /* 546 + * Compute the new pattern that we are going 547 + * to write to this location 548 + */ 549 + *pattern_ptr = compute_store_pattern(tid, w_ptr, cur_sweep_id); 550 + 551 + /* 552 + * STORE: Now let us write this pattern into 553 + * the location 554 + */ 555 + *w_ptr = *pattern_ptr; 556 + } 557 + } 558 + 559 + return NULL; 560 + } 561 + 562 + 563 + static unsigned long start_cpu = 0; 564 + static unsigned long nrthreads = 4; 565 + 566 + static pthread_t mem_snapshot_thread; 567 + 568 + static void *mem_snapshot_fn(void *arg) 569 + { 570 + int page_size = getpagesize(); 571 + size_t size = page_size; 572 + void *tmp = malloc(size); 573 + 574 + while (!corruption_found && !timeout) { 575 + /* Stop memory migration once corruption is found */ 576 + segv_wait = 1; 577 + 578 + mprotect(map1, size, PROT_READ); 579 + 580 + /* 581 + * Load from the working alias (map1). Loading from map2 582 + * also fails. 583 + */ 584 + memcpy(tmp, map1, size); 585 + 586 + /* 587 + * Stores must go via map2 which has write permissions, but 588 + * the corrupted data tends to be seen in the snapshot buffer, 589 + * so corruption does not appear to be introduced at the 590 + * copy-back via map2 alias here. 591 + */ 592 + memcpy(map2, tmp, size); 593 + /* 594 + * Before releasing other threads, must ensure the copy 595 + * back to 596 + */ 597 + asm volatile("sync" ::: "memory"); 598 + mprotect(map1, size, PROT_READ|PROT_WRITE); 599 + asm volatile("sync" ::: "memory"); 600 + segv_wait = 0; 601 + 602 + usleep(1); /* This value makes a big difference */ 603 + } 604 + 605 + return 0; 606 + } 607 + 608 + void alrm_sighandler(int sig) 609 + { 610 + timeout = 1; 611 + } 612 + 613 + int main(int argc, char *argv[]) 614 + { 615 + int c; 616 + int page_size = getpagesize(); 617 + time_t now; 618 + int i, dir_error; 619 + pthread_attr_t attr; 620 + key_t shm_key = (key_t) getpid(); 621 + int shmid, run_time = 20 * 60; 622 + struct sigaction sa_alrm; 623 + 624 + snprintf(logdir, LOGDIR_NAME_SIZE, 625 + "/tmp/logdir-%u", (unsigned int)getpid()); 626 + while ((c = getopt(argc, argv, "r:hn:l:t:")) != -1) { 627 + switch(c) { 628 + case 'r': 629 + start_cpu = strtoul(optarg, NULL, 10); 630 + break; 631 + case 'h': 632 + printf("%s [-r <start_cpu>] [-n <nrthreads>] [-l <logdir>] [-t <timeout>]\n", argv[0]); 633 + exit(0); 634 + break; 635 + case 'n': 636 + nrthreads = strtoul(optarg, NULL, 10); 637 + break; 638 + case 'l': 639 + strncpy(logdir, optarg, LOGDIR_NAME_SIZE); 640 + break; 641 + case 't': 642 + run_time = strtoul(optarg, NULL, 10); 643 + break; 644 + default: 645 + printf("invalid option\n"); 646 + exit(0); 647 + break; 648 + } 649 + } 650 + 651 + if (nrthreads > MAX_THREADS) 652 + nrthreads = MAX_THREADS; 653 + 654 + shmid = shmget(shm_key, page_size, IPC_CREAT|0666); 655 + if (shmid < 0) { 656 + err_msg("Failed shmget\n"); 657 + } 658 + 659 + map1 = shmat(shmid, NULL, 0); 660 + if (map1 == (void *) -1) { 661 + err_msg("Failed shmat"); 662 + } 663 + 664 + map2 = shmat(shmid, NULL, 0); 665 + if (map2 == (void *) -1) { 666 + err_msg("Failed shmat"); 667 + } 668 + 669 + dir_error = mkdir(logdir, 0755); 670 + 671 + if (dir_error) { 672 + err_msg("Failed mkdir"); 673 + } 674 + 675 + printf("start_cpu list:%lu\n", start_cpu); 676 + printf("number of worker threads:%lu + 1 snapshot thread\n", nrthreads); 677 + printf("Allocated address:0x%016lx + secondary map:0x%016lx\n", (unsigned long)map1, (unsigned long)map2); 678 + printf("logdir at : %s\n", logdir); 679 + printf("Timeout: %d seconds\n", run_time); 680 + 681 + time(&now); 682 + printf("=================================\n"); 683 + printf(" Starting Test\n"); 684 + printf(" %s", ctime(&now)); 685 + printf("=================================\n"); 686 + 687 + for (i = 0; i < nrthreads; i++) { 688 + if (1 && !fork()) { 689 + prctl(PR_SET_PDEATHSIG, SIGKILL); 690 + set_mycpu(start_cpu + i); 691 + for (;;) 692 + sched_yield(); 693 + exit(0); 694 + } 695 + } 696 + 697 + 698 + sa_alrm.sa_handler = &alrm_sighandler; 699 + sigemptyset(&sa_alrm.sa_mask); 700 + sa_alrm.sa_flags = 0; 701 + 702 + if (sigaction(SIGALRM, &sa_alrm, 0) == -1) { 703 + err_msg("Failed signal handler registration\n"); 704 + } 705 + 706 + alarm(run_time); 707 + 708 + pthread_attr_init(&attr); 709 + for (i = 0; i < nrthreads; i++) { 710 + rim_thread_ids[i] = i; 711 + pthread_create(&rim_threads[i], &attr, rim_fn, &rim_thread_ids[i]); 712 + set_pthread_cpu(rim_threads[i], start_cpu + i); 713 + } 714 + 715 + pthread_create(&mem_snapshot_thread, &attr, mem_snapshot_fn, map1); 716 + set_pthread_cpu(mem_snapshot_thread, start_cpu + i); 717 + 718 + 719 + pthread_join(mem_snapshot_thread, NULL); 720 + for (i = 0; i < nrthreads; i++) { 721 + pthread_join(rim_threads[i], NULL); 722 + } 723 + 724 + if (!timeout) { 725 + time(&now); 726 + printf("=================================\n"); 727 + printf(" Data Corruption Detected\n"); 728 + printf(" %s", ctime(&now)); 729 + printf(" See logfiles in %s\n", logdir); 730 + printf("=================================\n"); 731 + return 1; 732 + } 733 + return 0; 734 + }

+1

tools/testing/selftests/powerpc/tm/.gitignore

··· 17 17 tm-unavailable 18 18 tm-trap 19 19 tm-sigreturn 20 + tm-poison

+1 -1

tools/testing/selftests/powerpc/tm/Makefile

··· 5 5 TEST_GEN_PROGS := tm-resched-dscr tm-syscall tm-signal-msr-resv tm-signal-stack \ 6 6 tm-vmxcopy tm-fork tm-tar tm-tmspr tm-vmx-unavail tm-unavailable tm-trap \ 7 7 $(SIGNAL_CONTEXT_CHK_TESTS) tm-sigreturn tm-signal-sigreturn-nt \ 8 - tm-signal-context-force-tm 8 + tm-signal-context-force-tm tm-poison 9 9 10 10 top_srcdir = ../../../../.. 11 11 include ../../lib.mk

+179

tools/testing/selftests/powerpc/tm/tm-poison.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Copyright 2019, Gustavo Romero, Michael Neuling, IBM Corp. 4 + * 5 + * This test will spawn two processes. Both will be attached to the same 6 + * CPU (CPU 0). The child will be in a loop writing to FP register f31 and 7 + * VMX/VEC/Altivec register vr31 a known value, called poison, calling 8 + * sched_yield syscall after to allow the parent to switch on the CPU. 9 + * Parent will set f31 and vr31 to 1 and in a loop will check if f31 and 10 + * vr31 remain 1 as expected until a given timeout (2m). If the issue is 11 + * present child's poison will leak into parent's f31 or vr31 registers, 12 + * otherwise, poison will never leak into parent's f31 and vr31 registers. 13 + */ 14 + 15 + #define _GNU_SOURCE 16 + #include <stdio.h> 17 + #include <stdlib.h> 18 + #include <unistd.h> 19 + #include <inttypes.h> 20 + #include <sched.h> 21 + #include <sys/types.h> 22 + #include <signal.h> 23 + #include <inttypes.h> 24 + 25 + #include "tm.h" 26 + 27 + int tm_poison_test(void) 28 + { 29 + int pid; 30 + cpu_set_t cpuset; 31 + uint64_t poison = 0xdeadbeefc0dec0fe; 32 + uint64_t unknown = 0; 33 + bool fail_fp = false; 34 + bool fail_vr = false; 35 + 36 + SKIP_IF(!have_htm()); 37 + 38 + /* Attach both Child and Parent to CPU 0 */ 39 + CPU_ZERO(&cpuset); 40 + CPU_SET(0, &cpuset); 41 + sched_setaffinity(0, sizeof(cpuset), &cpuset); 42 + 43 + pid = fork(); 44 + if (!pid) { 45 + /** 46 + * child 47 + */ 48 + while (1) { 49 + sched_yield(); 50 + asm ( 51 + "mtvsrd 31, %[poison];" // f31 = poison 52 + "mtvsrd 63, %[poison];" // vr31 = poison 53 + 54 + : : [poison] "r" (poison) : ); 55 + } 56 + } 57 + 58 + /** 59 + * parent 60 + */ 61 + asm ( 62 + /* 63 + * Set r3, r4, and f31 to known value 1 before entering 64 + * in transaction. They won't be written after that. 65 + */ 66 + " li 3, 0x1 ;" 67 + " li 4, 0x1 ;" 68 + " mtvsrd 31, 4 ;" 69 + 70 + /* 71 + * The Time Base (TB) is a 64-bit counter register that is 72 + * independent of the CPU clock and which is incremented 73 + * at a frequency of 512000000 Hz, so every 1.953125ns. 74 + * So it's necessary 120s/0.000000001953125s = 61440000000 75 + * increments to get a 2 minutes timeout. Below we set that 76 + * value in r5 and then use r6 to track initial TB value, 77 + * updating TB values in r7 at every iteration and comparing it 78 + * to r6. When r7 (current) - r6 (initial) > 61440000000 we bail 79 + * out since for sure we spent already 2 minutes in the loop. 80 + * SPR 268 is the TB register. 81 + */ 82 + " lis 5, 14 ;" 83 + " ori 5, 5, 19996 ;" 84 + " sldi 5, 5, 16 ;" // r5 = 61440000000 85 + 86 + " mfspr 6, 268 ;" // r6 (TB initial) 87 + "1: mfspr 7, 268 ;" // r7 (TB current) 88 + " subf 7, 6, 7 ;" // r7 - r6 > 61440000000 ? 89 + " cmpd 7, 5 ;" 90 + " bgt 3f ;" // yes, exit 91 + 92 + /* 93 + * Main loop to check f31 94 + */ 95 + " tbegin. ;" // no, try again 96 + " beq 1b ;" // restart if no timeout 97 + " mfvsrd 3, 31 ;" // read f31 98 + " cmpd 3, 4 ;" // f31 == 1 ? 99 + " bne 2f ;" // broken :-( 100 + " tabort. 3 ;" // try another transaction 101 + "2: tend. ;" // commit transaction 102 + "3: mr %[unknown], 3 ;" // record r3 103 + 104 + : [unknown] "=r" (unknown) 105 + : 106 + : "cr0", "r3", "r4", "r5", "r6", "r7", "vs31" 107 + 108 + ); 109 + 110 + /* 111 + * On leak 'unknown' will contain 'poison' value from child, 112 + * otherwise (no leak) 'unknown' will contain the same value 113 + * as r3 before entering in transactional mode, i.e. 0x1. 114 + */ 115 + fail_fp = unknown != 0x1; 116 + if (fail_fp) 117 + printf("Unknown value %#"PRIx64" leaked into f31!\n", unknown); 118 + else 119 + printf("Good, no poison or leaked value into FP registers\n"); 120 + 121 + asm ( 122 + /* 123 + * Set r3, r4, and vr31 to known value 1 before entering 124 + * in transaction. They won't be written after that. 125 + */ 126 + " li 3, 0x1 ;" 127 + " li 4, 0x1 ;" 128 + " mtvsrd 63, 4 ;" 129 + 130 + " lis 5, 14 ;" 131 + " ori 5, 5, 19996 ;" 132 + " sldi 5, 5, 16 ;" // r5 = 61440000000 133 + 134 + " mfspr 6, 268 ;" // r6 (TB initial) 135 + "1: mfspr 7, 268 ;" // r7 (TB current) 136 + " subf 7, 6, 7 ;" // r7 - r6 > 61440000000 ? 137 + " cmpd 7, 5 ;" 138 + " bgt 3f ;" // yes, exit 139 + 140 + /* 141 + * Main loop to check vr31 142 + */ 143 + " tbegin. ;" // no, try again 144 + " beq 1b ;" // restart if no timeout 145 + " mfvsrd 3, 63 ;" // read vr31 146 + " cmpd 3, 4 ;" // vr31 == 1 ? 147 + " bne 2f ;" // broken :-( 148 + " tabort. 3 ;" // try another transaction 149 + "2: tend. ;" // commit transaction 150 + "3: mr %[unknown], 3 ;" // record r3 151 + 152 + : [unknown] "=r" (unknown) 153 + : 154 + : "cr0", "r3", "r4", "r5", "r6", "r7", "vs63" 155 + 156 + ); 157 + 158 + /* 159 + * On leak 'unknown' will contain 'poison' value from child, 160 + * otherwise (no leak) 'unknown' will contain the same value 161 + * as r3 before entering in transactional mode, i.e. 0x1. 162 + */ 163 + fail_vr = unknown != 0x1; 164 + if (fail_vr) 165 + printf("Unknown value %#"PRIx64" leaked into vr31!\n", unknown); 166 + else 167 + printf("Good, no poison or leaked value into VEC registers\n"); 168 + 169 + kill(pid, SIGKILL); 170 + 171 + return (fail_fp | fail_vr); 172 + } 173 + 174 + int main(int argc, char *argv[]) 175 + { 176 + /* Test completes in about 4m */ 177 + test_harness_set_timeout(250); 178 + return test_harness(tm_poison_test, "tm_poison_test"); 179 + }

Configure Feed

Configure Feed