Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'riscv-for-linus-6.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux
Pull RISC-V updates from Palmer Dabbelt:
- Support for cbo.zero in userspace
- Support for CBOs on ACPI-based systems
- A handful of improvements for the T-Head cache flushing ops
- Support for software shadow call stacks
- Various cleanups and fixes
* tag 'riscv-for-linus-6.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (31 commits)
RISC-V: hwprobe: Fix vDSO SIGSEGV
riscv: configs: defconfig: Enable configs required for RZ/Five SoC
riscv: errata: prefix T-Head mnemonics with th.
riscv: put interrupt entries into .irqentry.text
riscv: mm: Update the comment of CONFIG_PAGE_OFFSET
riscv: Using TOOLCHAIN_HAS_ZIHINTPAUSE marco replace zihintpause
riscv/mm: Fix the comment for swap pte format
RISC-V: clarify the QEMU workaround in ISA parser
riscv: correct pt_level name via pgtable_l5/4_enabled
RISC-V: Provide pgtable_l5_enabled on rv32
clocksource: timer-riscv: Increase rating of clock_event_device for Sstc
clocksource: timer-riscv: Don't enable/disable timer interrupt
lkdtm: Fix CFI_BACKWARD on RISC-V
riscv: Use separate IRQ shadow call stacks
riscv: Implement Shadow Call Stack
riscv: Move global pointer loading to a macro
riscv: Deduplicate IRQ stack switching
riscv: VMAP_STACK overflow detection thread-safe
RISC-V: cacheflush: Initialize CBO variables on ACPI systems
RISC-V: ACPI: RHCT: Add function to get CBO block sizes
...
+918
-298
+6
Documentation/arch/riscv/hwprobe.rst
+6
Documentation/arch/riscv/hwprobe.rst
···
77
77
* :c:macro:`RISCV_HWPROBE_EXT_ZBS`: The Zbs extension is supported, as defined
78
78
in version 1.0 of the Bit-Manipulation ISA extensions.
79
79
80
+
* :c:macro:`RISCV_HWPROBE_EXT_ZICBOZ`: The Zicboz extension is supported, as
81
+
ratified in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs.
82
+
80
83
* :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance
81
84
information about the selected set of processors.
82
85
···
99
96
100
97
* :c:macro:`RISCV_HWPROBE_MISALIGNED_UNSUPPORTED`: Misaligned accesses are
101
98
not supported at all and will generate a misaligned address fault.
99
+
100
+
* :c:macro:`RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE`: An unsigned int which
101
+
represents the size of the Zicboz block in bytes.
+7
arch/riscv/Kconfig
+7
arch/riscv/Kconfig
···
39
39
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
40
40
select ARCH_HAS_UBSAN_SANITIZE_ALL
41
41
select ARCH_HAS_VDSO_DATA
42
+
select ARCH_KEEP_MEMBLOCK if ACPI
42
43
select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
43
44
select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT
44
45
select ARCH_STACKWALK
···
49
48
select ARCH_SUPPORTS_HUGETLBFS if MMU
50
49
select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
51
50
select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
51
+
select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
52
52
select ARCH_USE_MEMTEST
53
53
select ARCH_USE_QUEUED_RWLOCKS
54
54
select ARCH_USES_CFI_TRAPS if CFI_CLANG
···
175
173
config GCC_SUPPORTS_DYNAMIC_FTRACE
176
174
def_bool CC_IS_GCC
177
175
depends on $(cc-option,-fpatchable-function-entry=8)
176
+
177
+
config HAVE_SHADOW_CALL_STACK
178
+
def_bool $(cc-option,-fsanitize=shadow-call-stack)
179
+
# https://github.com/riscv-non-isa/riscv-elf-psabi-doc/commit/a484e843e6eeb51f0cb7b8819e50da6d2444d769
180
+
depends on $(ld-option,--no-relax-gp)
178
181
179
182
config ARCH_MMAP_RND_BITS_MIN
180
183
default 18 if 64BIT
+4
arch/riscv/Makefile
+4
arch/riscv/Makefile
+52
arch/riscv/configs/defconfig
+52
arch/riscv/configs/defconfig
···
37
37
CONFIG_HOTPLUG_CPU=y
38
38
CONFIG_PM=y
39
39
CONFIG_CPU_IDLE=y
40
+
CONFIG_CPU_FREQ=y
41
+
CONFIG_CPU_FREQ_STAT=y
42
+
CONFIG_CPU_FREQ_GOV_POWERSAVE=m
43
+
CONFIG_CPU_FREQ_GOV_USERSPACE=y
44
+
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
45
+
CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m
46
+
CONFIG_CPUFREQ_DT=y
40
47
CONFIG_VIRTUALIZATION=y
41
48
CONFIG_KVM=m
42
49
CONFIG_ACPI=y
···
102
95
CONFIG_CGROUP_NET_PRIO=y
103
96
CONFIG_NET_9P=y
104
97
CONFIG_NET_9P_VIRTIO=y
98
+
CONFIG_CAN=m
105
99
CONFIG_PCI=y
106
100
CONFIG_PCIEPORTBUS=y
107
101
CONFIG_PCI_HOST_GENERIC=y
···
110
102
CONFIG_PCIE_FU740=y
111
103
CONFIG_DEVTMPFS=y
112
104
CONFIG_DEVTMPFS_MOUNT=y
105
+
CONFIG_MTD=y
106
+
CONFIG_MTD_BLOCK=y
107
+
CONFIG_MTD_CFI=y
108
+
CONFIG_MTD_CFI_ADV_OPTIONS=y
109
+
CONFIG_MTD_SPI_NOR=y
113
110
CONFIG_BLK_DEV_LOOP=y
114
111
CONFIG_VIRTIO_BLK=y
115
112
CONFIG_BLK_DEV_NVME=m
···
137
124
CONFIG_MACB=y
138
125
CONFIG_E1000E=y
139
126
CONFIG_R8169=y
127
+
CONFIG_RAVB=y
140
128
CONFIG_STMMAC_ETH=m
129
+
CONFIG_MICREL_PHY=y
141
130
CONFIG_MICROSEMI_PHY=y
131
+
CONFIG_CAN_RCAR_CANFD=m
142
132
CONFIG_INPUT_MOUSEDEV=y
143
133
CONFIG_KEYBOARD_SUN4I_LRADC=m
144
134
CONFIG_SERIAL_8250=y
···
152
136
CONFIG_VIRTIO_CONSOLE=y
153
137
CONFIG_HW_RANDOM=y
154
138
CONFIG_HW_RANDOM_VIRTIO=y
139
+
CONFIG_I2C_CHARDEV=m
155
140
CONFIG_I2C_MV64XXX=m
141
+
CONFIG_I2C_RIIC=y
156
142
CONFIG_SPI=y
143
+
CONFIG_SPI_RSPI=m
157
144
CONFIG_SPI_SIFIVE=y
158
145
CONFIG_SPI_SUN6I=y
159
146
# CONFIG_PTP_1588_CLOCK is not set
160
147
CONFIG_GPIO_SIFIVE=y
148
+
CONFIG_CPU_THERMAL=y
149
+
CONFIG_DEVFREQ_THERMAL=y
150
+
CONFIG_RZG2L_THERMAL=y
161
151
CONFIG_WATCHDOG=y
162
152
CONFIG_SUNXI_WATCHDOG=y
153
+
CONFIG_RENESAS_RZG2LWDT=y
163
154
CONFIG_REGULATOR=y
164
155
CONFIG_REGULATOR_FIXED_VOLTAGE=y
156
+
CONFIG_REGULATOR_GPIO=y
165
157
CONFIG_DRM=m
166
158
CONFIG_DRM_RADEON=m
167
159
CONFIG_DRM_NOUVEAU=m
···
177
153
CONFIG_DRM_VIRTIO_GPU=m
178
154
CONFIG_FB=y
179
155
CONFIG_FRAMEBUFFER_CONSOLE=y
156
+
CONFIG_SOUND=y
157
+
CONFIG_SND=y
158
+
CONFIG_SND_SOC=y
159
+
CONFIG_SND_SOC_RZ=m
160
+
CONFIG_SND_SOC_WM8978=m
161
+
CONFIG_SND_SIMPLE_CARD=m
180
162
CONFIG_USB=y
163
+
CONFIG_USB_OTG=y
181
164
CONFIG_USB_XHCI_HCD=y
182
165
CONFIG_USB_XHCI_PLATFORM=y
183
166
CONFIG_USB_EHCI_HCD=y
184
167
CONFIG_USB_EHCI_HCD_PLATFORM=y
185
168
CONFIG_USB_OHCI_HCD=y
186
169
CONFIG_USB_OHCI_HCD_PLATFORM=y
170
+
CONFIG_USB_RENESAS_USBHS=m
187
171
CONFIG_USB_STORAGE=y
188
172
CONFIG_USB_UAS=y
189
173
CONFIG_USB_MUSB_HDRC=m
190
174
CONFIG_USB_MUSB_SUNXI=m
191
175
CONFIG_NOP_USB_XCEIV=m
176
+
CONFIG_USB_GADGET=y
177
+
CONFIG_USB_RENESAS_USBHS_UDC=m
178
+
CONFIG_USB_CONFIGFS=m
179
+
CONFIG_USB_CONFIGFS_SERIAL=y
180
+
CONFIG_USB_CONFIGFS_ACM=y
181
+
CONFIG_USB_CONFIGFS_OBEX=y
182
+
CONFIG_USB_CONFIGFS_NCM=y
183
+
CONFIG_USB_CONFIGFS_ECM=y
184
+
CONFIG_USB_CONFIGFS_ECM_SUBSET=y
185
+
CONFIG_USB_CONFIGFS_RNDIS=y
186
+
CONFIG_USB_CONFIGFS_EEM=y
187
+
CONFIG_USB_CONFIGFS_MASS_STORAGE=y
188
+
CONFIG_USB_CONFIGFS_F_FS=y
192
189
CONFIG_MMC=y
193
190
CONFIG_MMC_SDHCI=y
194
191
CONFIG_MMC_SDHCI_PLTFM=y
195
192
CONFIG_MMC_SDHCI_CADENCE=y
196
193
CONFIG_MMC_SPI=y
194
+
CONFIG_MMC_SDHI=y
197
195
CONFIG_MMC_SUNXI=y
198
196
CONFIG_RTC_CLASS=y
199
197
CONFIG_RTC_DRV_SUN6I=y
200
198
CONFIG_DMADEVICES=y
201
199
CONFIG_DMA_SUN6I=m
200
+
CONFIG_RZ_DMAC=y
202
201
CONFIG_VIRTIO_PCI=y
203
202
CONFIG_VIRTIO_BALLOON=y
204
203
CONFIG_VIRTIO_INPUT=y
205
204
CONFIG_VIRTIO_MMIO=y
205
+
CONFIG_RENESAS_OSTM=y
206
206
CONFIG_SUN8I_DE2_CCU=m
207
207
CONFIG_SUN50I_IOMMU=y
208
208
CONFIG_RPMSG_CHAR=y
209
209
CONFIG_RPMSG_CTRL=y
210
210
CONFIG_RPMSG_VIRTIO=y
211
211
CONFIG_ARCH_R9A07G043=y
212
+
CONFIG_IIO=y
213
+
CONFIG_RZG2L_ADC=m
214
+
CONFIG_RESET_RZG2L_USBPHY_CTRL=y
212
215
CONFIG_PHY_SUN4I_USB=m
216
+
CONFIG_PHY_RCAR_GEN3_USB2=y
213
217
CONFIG_LIBNVDIMM=y
214
218
CONFIG_NVMEM_SUNXI_SID=y
215
219
CONFIG_EXT4_FS=y
+6
arch/riscv/include/asm/acpi.h
+6
arch/riscv/include/asm/acpi.h
···
66
66
unsigned int cpu, const char **isa);
67
67
68
68
static inline int acpi_numa_get_nid(unsigned int cpu) { return NUMA_NO_NODE; }
69
+
void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size,
70
+
u32 *cboz_size, u32 *cbop_size);
69
71
#else
70
72
static inline void acpi_init_rintc_map(void) { }
71
73
static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu)
···
80
78
{
81
79
return -EINVAL;
82
80
}
81
+
82
+
static inline void acpi_get_cbo_block_size(struct acpi_table_header *table,
83
+
u32 *cbom_size, u32 *cboz_size,
84
+
u32 *cbop_size) { }
83
85
84
86
#endif /* CONFIG_ACPI */
85
87
-1
arch/riscv/include/asm/asm-prototypes.h
-1
arch/riscv/include/asm/asm-prototypes.h
···
25
25
DECLARE_DO_ERROR_INFO(do_trap_ecall_m);
26
26
DECLARE_DO_ERROR_INFO(do_trap_break);
27
27
28
-
asmlinkage unsigned long get_overflow_stack(void);
29
28
asmlinkage void handle_bad_stack(struct pt_regs *regs);
30
29
asmlinkage void do_page_fault(struct pt_regs *regs);
31
30
asmlinkage void do_irq(struct pt_regs *regs);
+41
arch/riscv/include/asm/asm.h
+41
arch/riscv/include/asm/asm.h
···
82
82
.endr
83
83
.endm
84
84
85
+
#ifdef CONFIG_SMP
86
+
#ifdef CONFIG_32BIT
87
+
#define PER_CPU_OFFSET_SHIFT 2
88
+
#else
89
+
#define PER_CPU_OFFSET_SHIFT 3
90
+
#endif
91
+
92
+
.macro asm_per_cpu dst sym tmp
93
+
REG_L \tmp, TASK_TI_CPU_NUM(tp)
94
+
slli \tmp, \tmp, PER_CPU_OFFSET_SHIFT
95
+
la \dst, __per_cpu_offset
96
+
add \dst, \dst, \tmp
97
+
REG_L \tmp, 0(\dst)
98
+
la \dst, \sym
99
+
add \dst, \dst, \tmp
100
+
.endm
101
+
#else /* CONFIG_SMP */
102
+
.macro asm_per_cpu dst sym tmp
103
+
la \dst, \sym
104
+
.endm
105
+
#endif /* CONFIG_SMP */
106
+
107
+
.macro load_per_cpu dst ptr tmp
108
+
asm_per_cpu \dst \ptr \tmp
109
+
REG_L \dst, 0(\dst)
110
+
.endm
111
+
112
+
#ifdef CONFIG_SHADOW_CALL_STACK
113
+
/* gp is used as the shadow call stack pointer instead */
114
+
.macro load_global_pointer
115
+
.endm
116
+
#else
117
+
/* load __global_pointer to gp */
118
+
.macro load_global_pointer
119
+
.option push
120
+
.option norelax
121
+
la gp, __global_pointer$
122
+
.option pop
123
+
.endm
124
+
#endif /* CONFIG_SHADOW_CALL_STACK */
125
+
85
126
/* save all GPs except x1 ~ x5 */
86
127
.macro save_from_x6_to_x31
87
128
REG_S x6, PT_T1(sp)
+1
arch/riscv/include/asm/cpufeature.h
+1
arch/riscv/include/asm/cpufeature.h
+7
-7
arch/riscv/include/asm/errata_list.h
+7
-7
arch/riscv/include/asm/errata_list.h
···
95
95
#endif
96
96
97
97
/*
98
-
* dcache.ipa rs1 (invalidate, physical address)
98
+
* th.dcache.ipa rs1 (invalidate, physical address)
99
99
* | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
100
100
* 0000001 01010 rs1 000 00000 0001011
101
-
* dache.iva rs1 (invalida, virtual address)
101
+
* th.dache.iva rs1 (invalida, virtual address)
102
102
* 0000001 00110 rs1 000 00000 0001011
103
103
*
104
-
* dcache.cpa rs1 (clean, physical address)
104
+
* th.dcache.cpa rs1 (clean, physical address)
105
105
* | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
106
106
* 0000001 01001 rs1 000 00000 0001011
107
-
* dcache.cva rs1 (clean, virtual address)
107
+
* th.dcache.cva rs1 (clean, virtual address)
108
108
* 0000001 00101 rs1 000 00000 0001011
109
109
*
110
-
* dcache.cipa rs1 (clean then invalidate, physical address)
110
+
* th.dcache.cipa rs1 (clean then invalidate, physical address)
111
111
* | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
112
112
* 0000001 01011 rs1 000 00000 0001011
113
-
* dcache.civa rs1 (... virtual address)
113
+
* th.dcache.civa rs1 (... virtual address)
114
114
* 0000001 00111 rs1 000 00000 0001011
115
115
*
116
-
* sync.s (make sure all cache operations finished)
116
+
* th.sync.s (make sure all cache operations finished)
117
117
* | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
118
118
* 0000000 11001 00000 000 00000 0001011
119
119
*/
+16
arch/riscv/include/asm/hwcap.h
+16
arch/riscv/include/asm/hwcap.h
···
72
72
#ifndef __ASSEMBLY__
73
73
74
74
#include <linux/jump_label.h>
75
+
#include <asm/cpufeature.h>
75
76
76
77
unsigned long riscv_get_elf_hwcap(void);
77
78
···
140
139
return true;
141
140
}
142
141
142
+
static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
143
+
{
144
+
if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && riscv_has_extension_likely(ext))
145
+
return true;
146
+
147
+
return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
148
+
}
149
+
150
+
static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext)
151
+
{
152
+
if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && riscv_has_extension_unlikely(ext))
153
+
return true;
154
+
155
+
return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
156
+
}
143
157
#endif
144
158
145
159
#endif /* _ASM_RISCV_HWCAP_H */
+6
-1
arch/riscv/include/asm/hwprobe.h
+6
-1
arch/riscv/include/asm/hwprobe.h
+3
arch/riscv/include/asm/irq_stack.h
+3
arch/riscv/include/asm/irq_stack.h
+2
-2
arch/riscv/include/asm/page.h
+2
-2
arch/riscv/include/asm/page.h
···
33
33
#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL)
34
34
#endif
35
35
/*
36
-
* By default, CONFIG_PAGE_OFFSET value corresponds to SV48 address space so
37
-
* define the PAGE_OFFSET value for SV39.
36
+
* By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so
37
+
* define the PAGE_OFFSET value for SV48 and SV39.
38
38
*/
39
39
#define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL)
40
40
#define PAGE_OFFSET_L3 _AC(0xffffffd800000000, UL)
+3
arch/riscv/include/asm/pgtable-32.h
+3
arch/riscv/include/asm/pgtable-32.h
+1
-2
arch/riscv/include/asm/pgtable.h
+1
-2
arch/riscv/include/asm/pgtable.h
···
811
811
* bit 5: _PAGE_PROT_NONE (zero)
812
812
* bit 6: exclusive marker
813
813
* bits 7 to 11: swap type
814
-
* bits 11 to XLEN-1: swap offset
814
+
* bits 12 to XLEN-1: swap offset
815
815
*/
816
816
#define __SWP_TYPE_SHIFT 7
817
817
#define __SWP_TYPE_BITS 5
···
914
914
#define dtb_early_pa _dtb_early_pa
915
915
#endif /* CONFIG_XIP_KERNEL */
916
916
extern u64 satp_mode;
917
-
extern bool pgtable_l4_enabled;
918
917
919
918
void paging_init(void);
920
919
void misc_mem_init(void);
+54
arch/riscv/include/asm/scs.h
+54
arch/riscv/include/asm/scs.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef _ASM_SCS_H
3
+
#define _ASM_SCS_H
4
+
5
+
#ifdef __ASSEMBLY__
6
+
#include <asm/asm-offsets.h>
7
+
8
+
#ifdef CONFIG_SHADOW_CALL_STACK
9
+
10
+
/* Load init_shadow_call_stack to gp. */
11
+
.macro scs_load_init_stack
12
+
la gp, init_shadow_call_stack
13
+
XIP_FIXUP_OFFSET gp
14
+
.endm
15
+
16
+
/* Load the per-CPU IRQ shadow call stack to gp. */
17
+
.macro scs_load_irq_stack tmp
18
+
load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp
19
+
.endm
20
+
21
+
/* Load task_scs_sp(current) to gp. */
22
+
.macro scs_load_current
23
+
REG_L gp, TASK_TI_SCS_SP(tp)
24
+
.endm
25
+
26
+
/* Load task_scs_sp(current) to gp, but only if tp has changed. */
27
+
.macro scs_load_current_if_task_changed prev
28
+
beq \prev, tp, _skip_scs
29
+
scs_load_current
30
+
_skip_scs:
31
+
.endm
32
+
33
+
/* Save gp to task_scs_sp(current). */
34
+
.macro scs_save_current
35
+
REG_S gp, TASK_TI_SCS_SP(tp)
36
+
.endm
37
+
38
+
#else /* CONFIG_SHADOW_CALL_STACK */
39
+
40
+
.macro scs_load_init_stack
41
+
.endm
42
+
.macro scs_load_irq_stack tmp
43
+
.endm
44
+
.macro scs_load_current
45
+
.endm
46
+
.macro scs_load_current_if_task_changed prev
47
+
.endm
48
+
.macro scs_save_current
49
+
.endm
50
+
51
+
#endif /* CONFIG_SHADOW_CALL_STACK */
52
+
#endif /* __ASSEMBLY__ */
53
+
54
+
#endif /* _ASM_SCS_H */
+13
-3
arch/riscv/include/asm/thread_info.h
+13
-3
arch/riscv/include/asm/thread_info.h
···
34
34
35
35
#ifndef __ASSEMBLY__
36
36
37
-
extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)];
38
-
extern unsigned long spin_shadow_stack;
39
-
40
37
#include <asm/processor.h>
41
38
#include <asm/csr.h>
42
39
···
57
60
long user_sp; /* User stack pointer */
58
61
int cpu;
59
62
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
63
+
#ifdef CONFIG_SHADOW_CALL_STACK
64
+
void *scs_base;
65
+
void *scs_sp;
66
+
#endif
60
67
};
68
+
69
+
#ifdef CONFIG_SHADOW_CALL_STACK
70
+
#define INIT_SCS \
71
+
.scs_base = init_shadow_call_stack, \
72
+
.scs_sp = init_shadow_call_stack,
73
+
#else
74
+
#define INIT_SCS
75
+
#endif
61
76
62
77
/*
63
78
* macros/functions for gaining access to the thread information structure
···
80
71
{ \
81
72
.flags = 0, \
82
73
.preempt_count = INIT_PREEMPT_COUNT, \
74
+
INIT_SCS \
83
75
}
84
76
85
77
void arch_release_task_struct(struct task_struct *tsk);
+1
-1
arch/riscv/include/asm/vdso/processor.h
+1
-1
arch/riscv/include/asm/vdso/processor.h
+2
arch/riscv/include/uapi/asm/hwprobe.h
+2
arch/riscv/include/uapi/asm/hwprobe.h
···
29
29
#define RISCV_HWPROBE_EXT_ZBA (1 << 3)
30
30
#define RISCV_HWPROBE_EXT_ZBB (1 << 4)
31
31
#define RISCV_HWPROBE_EXT_ZBS (1 << 5)
32
+
#define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6)
32
33
#define RISCV_HWPROBE_KEY_CPUPERF_0 5
33
34
#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
34
35
#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
···
37
36
#define RISCV_HWPROBE_MISALIGNED_FAST (3 << 0)
38
37
#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0)
39
38
#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0)
39
+
#define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6
40
40
/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
41
41
42
42
#endif
+86
-3
arch/riscv/kernel/acpi.c
+86
-3
arch/riscv/kernel/acpi.c
···
14
14
*/
15
15
16
16
#include <linux/acpi.h>
17
-
#include <linux/io.h>
18
-
#include <linux/pci.h>
19
17
#include <linux/efi.h>
18
+
#include <linux/io.h>
19
+
#include <linux/memblock.h>
20
+
#include <linux/pci.h>
20
21
21
22
int acpi_noirq = 1; /* skip ACPI IRQ initialization */
22
23
int acpi_disabled = 1;
···
218
217
219
218
void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size)
220
219
{
221
-
return (void __iomem *)memremap(phys, size, MEMREMAP_WB);
220
+
efi_memory_desc_t *md, *region = NULL;
221
+
pgprot_t prot;
222
+
223
+
if (WARN_ON_ONCE(!efi_enabled(EFI_MEMMAP)))
224
+
return NULL;
225
+
226
+
for_each_efi_memory_desc(md) {
227
+
u64 end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
228
+
229
+
if (phys < md->phys_addr || phys >= end)
230
+
continue;
231
+
232
+
if (phys + size > end) {
233
+
pr_warn(FW_BUG "requested region covers multiple EFI memory regions\n");
234
+
return NULL;
235
+
}
236
+
region = md;
237
+
break;
238
+
}
239
+
240
+
/*
241
+
* It is fine for AML to remap regions that are not represented in the
242
+
* EFI memory map at all, as it only describes normal memory, and MMIO
243
+
* regions that require a virtual mapping to make them accessible to
244
+
* the EFI runtime services.
245
+
*/
246
+
prot = PAGE_KERNEL_IO;
247
+
if (region) {
248
+
switch (region->type) {
249
+
case EFI_LOADER_CODE:
250
+
case EFI_LOADER_DATA:
251
+
case EFI_BOOT_SERVICES_CODE:
252
+
case EFI_BOOT_SERVICES_DATA:
253
+
case EFI_CONVENTIONAL_MEMORY:
254
+
case EFI_PERSISTENT_MEMORY:
255
+
if (memblock_is_map_memory(phys) ||
256
+
!memblock_is_region_memory(phys, size)) {
257
+
pr_warn(FW_BUG "requested region covers kernel memory\n");
258
+
return NULL;
259
+
}
260
+
261
+
/*
262
+
* Mapping kernel memory is permitted if the region in
263
+
* question is covered by a single memblock with the
264
+
* NOMAP attribute set: this enables the use of ACPI
265
+
* table overrides passed via initramfs.
266
+
* This particular use case only requires read access.
267
+
*/
268
+
fallthrough;
269
+
270
+
case EFI_RUNTIME_SERVICES_CODE:
271
+
/*
272
+
* This would be unusual, but not problematic per se,
273
+
* as long as we take care not to create a writable
274
+
* mapping for executable code.
275
+
*/
276
+
prot = PAGE_KERNEL_RO;
277
+
break;
278
+
279
+
case EFI_ACPI_RECLAIM_MEMORY:
280
+
/*
281
+
* ACPI reclaim memory is used to pass firmware tables
282
+
* and other data that is intended for consumption by
283
+
* the OS only, which may decide it wants to reclaim
284
+
* that memory and use it for something else. We never
285
+
* do that, but we usually add it to the linear map
286
+
* anyway, in which case we should use the existing
287
+
* mapping.
288
+
*/
289
+
if (memblock_is_map_memory(phys))
290
+
return (void __iomem *)__va(phys);
291
+
fallthrough;
292
+
293
+
default:
294
+
if (region->attribute & EFI_MEMORY_WB)
295
+
prot = PAGE_KERNEL;
296
+
else if ((region->attribute & EFI_MEMORY_WC) ||
297
+
(region->attribute & EFI_MEMORY_WT))
298
+
prot = pgprot_writecombine(PAGE_KERNEL);
299
+
}
300
+
}
301
+
302
+
return ioremap_prot(phys, size, pgprot_val(prot));
222
303
}
223
304
224
305
#ifdef CONFIG_PCI
+9
arch/riscv/kernel/asm-offsets.c
+9
arch/riscv/kernel/asm-offsets.c
···
14
14
#include <asm/thread_info.h>
15
15
#include <asm/ptrace.h>
16
16
#include <asm/cpu_ops_sbi.h>
17
+
#include <asm/stacktrace.h>
17
18
#include <asm/suspend.h>
18
19
19
20
void asm_offsets(void);
···
39
38
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
40
39
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
41
40
OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
41
+
#ifdef CONFIG_SHADOW_CALL_STACK
42
+
OFFSET(TASK_TI_SCS_SP, task_struct, thread_info.scs_sp);
43
+
#endif
42
44
45
+
OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu);
43
46
OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]);
44
47
OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]);
45
48
OFFSET(TASK_THREAD_F2, task_struct, thread.fstate.f[2]);
···
484
479
OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr);
485
480
OFFSET(SBI_HART_BOOT_TASK_PTR_OFFSET, sbi_hart_boot_data, task_ptr);
486
481
OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr);
482
+
483
+
DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN));
484
+
OFFSET(STACKFRAME_FP, stackframe, fp);
485
+
OFFSET(STACKFRAME_RA, stackframe, ra);
487
486
}
+12
-5
arch/riscv/kernel/cpufeature.c
+12
-5
arch/riscv/kernel/cpufeature.c
···
93
93
return true;
94
94
case RISCV_ISA_EXT_ZICBOZ:
95
95
if (!riscv_cboz_block_size) {
96
-
pr_err("Zicboz detected in ISA string, but no cboz-block-size found\n");
96
+
pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n");
97
97
return false;
98
98
} else if (!is_power_of_2(riscv_cboz_block_size)) {
99
-
pr_err("cboz-block-size present, but is not a power-of-2\n");
99
+
pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
100
100
return false;
101
101
}
102
102
return true;
···
206
206
switch (*ext) {
207
207
case 's':
208
208
/*
209
-
* Workaround for invalid single-letter 's' & 'u'(QEMU).
209
+
* Workaround for invalid single-letter 's' & 'u' (QEMU).
210
210
* No need to set the bit in riscv_isa as 's' & 'u' are
211
-
* not valid ISA extensions. It works until multi-letter
212
-
* extension starting with "Su" appears.
211
+
* not valid ISA extensions. It works unless the first
212
+
* multi-letter extension in the ISA string begins with
213
+
* "Su" and is not prefixed with an underscore.
213
214
*/
214
215
if (ext[-1] != '_' && ext[1] == 'u') {
215
216
++isa;
···
655
654
}
656
655
657
656
arch_initcall(check_unaligned_access_boot_cpu);
657
+
658
+
void riscv_user_isa_enable(void)
659
+
{
660
+
if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ))
661
+
csr_set(CSR_SENVCFG, ENVCFG_CBZE);
662
+
}
658
663
659
664
#ifdef CONFIG_RISCV_ALTERNATIVE
660
665
/*
+62
-64
arch/riscv/kernel/entry.S
+62
-64
arch/riscv/kernel/entry.S
···
9
9
10
10
#include <asm/asm.h>
11
11
#include <asm/csr.h>
12
+
#include <asm/scs.h>
12
13
#include <asm/unistd.h>
14
+
#include <asm/page.h>
13
15
#include <asm/thread_info.h>
14
16
#include <asm/asm-offsets.h>
15
17
#include <asm/errata_list.h>
18
+
#include <linux/sizes.h>
19
+
20
+
.section .irqentry.text, "ax"
16
21
17
22
SYM_CODE_START(handle_exception)
18
23
/*
···
78
73
csrw CSR_SCRATCH, x0
79
74
80
75
/* Load the global pointer */
81
-
.option push
82
-
.option norelax
83
-
la gp, __global_pointer$
84
-
.option pop
76
+
load_global_pointer
77
+
78
+
/* Load the kernel shadow call stack pointer if coming from userspace */
79
+
scs_load_current_if_task_changed s5
80
+
85
81
move a0, sp /* pt_regs */
86
82
la ra, ret_from_exception
87
83
···
128
122
/* Save unwound kernel stack pointer in thread_info */
129
123
addi s0, sp, PT_SIZE_ON_STACK
130
124
REG_S s0, TASK_TI_KERNEL_SP(tp)
125
+
126
+
/* Save the kernel shadow call stack pointer */
127
+
scs_save_current
131
128
132
129
/*
133
130
* Save TP into the scratch register , so we can find the kernel data
···
179
170
180
171
#ifdef CONFIG_VMAP_STACK
181
172
SYM_CODE_START_LOCAL(handle_kernel_stack_overflow)
182
-
/*
183
-
* Takes the psuedo-spinlock for the shadow stack, in case multiple
184
-
* harts are concurrently overflowing their kernel stacks. We could
185
-
* store any value here, but since we're overflowing the kernel stack
186
-
* already we only have SP to use as a scratch register. So we just
187
-
* swap in the address of the spinlock, as that's definately non-zero.
188
-
*
189
-
* Pairs with a store_release in handle_bad_stack().
190
-
*/
191
-
1: la sp, spin_shadow_stack
192
-
REG_AMOSWAP_AQ sp, sp, (sp)
193
-
bnez sp, 1b
173
+
/* we reach here from kernel context, sscratch must be 0 */
174
+
csrrw x31, CSR_SCRATCH, x31
175
+
asm_per_cpu sp, overflow_stack, x31
176
+
li x31, OVERFLOW_STACK_SIZE
177
+
add sp, sp, x31
178
+
/* zero out x31 again and restore x31 */
179
+
xor x31, x31, x31
180
+
csrrw x31, CSR_SCRATCH, x31
194
181
195
-
la sp, shadow_stack
196
-
addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE
197
-
198
-
//save caller register to shadow stack
199
-
addi sp, sp, -(PT_SIZE_ON_STACK)
200
-
REG_S x1, PT_RA(sp)
201
-
REG_S x5, PT_T0(sp)
202
-
REG_S x6, PT_T1(sp)
203
-
REG_S x7, PT_T2(sp)
204
-
REG_S x10, PT_A0(sp)
205
-
REG_S x11, PT_A1(sp)
206
-
REG_S x12, PT_A2(sp)
207
-
REG_S x13, PT_A3(sp)
208
-
REG_S x14, PT_A4(sp)
209
-
REG_S x15, PT_A5(sp)
210
-
REG_S x16, PT_A6(sp)
211
-
REG_S x17, PT_A7(sp)
212
-
REG_S x28, PT_T3(sp)
213
-
REG_S x29, PT_T4(sp)
214
-
REG_S x30, PT_T5(sp)
215
-
REG_S x31, PT_T6(sp)
216
-
217
-
la ra, restore_caller_reg
218
-
tail get_overflow_stack
219
-
220
-
restore_caller_reg:
221
-
//save per-cpu overflow stack
222
-
REG_S a0, -8(sp)
223
-
//restore caller register from shadow_stack
224
-
REG_L x1, PT_RA(sp)
225
-
REG_L x5, PT_T0(sp)
226
-
REG_L x6, PT_T1(sp)
227
-
REG_L x7, PT_T2(sp)
228
-
REG_L x10, PT_A0(sp)
229
-
REG_L x11, PT_A1(sp)
230
-
REG_L x12, PT_A2(sp)
231
-
REG_L x13, PT_A3(sp)
232
-
REG_L x14, PT_A4(sp)
233
-
REG_L x15, PT_A5(sp)
234
-
REG_L x16, PT_A6(sp)
235
-
REG_L x17, PT_A7(sp)
236
-
REG_L x28, PT_T3(sp)
237
-
REG_L x29, PT_T4(sp)
238
-
REG_L x30, PT_T5(sp)
239
-
REG_L x31, PT_T6(sp)
240
-
241
-
//load per-cpu overflow stack
242
-
REG_L sp, -8(sp)
243
182
addi sp, sp, -(PT_SIZE_ON_STACK)
244
183
245
184
//save context to overflow stack
···
225
268
tail syscall_exit_to_user_mode
226
269
SYM_CODE_END(ret_from_fork)
227
270
271
+
#ifdef CONFIG_IRQ_STACKS
272
+
/*
273
+
* void call_on_irq_stack(struct pt_regs *regs,
274
+
* void (*func)(struct pt_regs *));
275
+
*
276
+
* Calls func(regs) using the per-CPU IRQ stack.
277
+
*/
278
+
SYM_FUNC_START(call_on_irq_stack)
279
+
/* Create a frame record to save ra and s0 (fp) */
280
+
addi sp, sp, -STACKFRAME_SIZE_ON_STACK
281
+
REG_S ra, STACKFRAME_RA(sp)
282
+
REG_S s0, STACKFRAME_FP(sp)
283
+
addi s0, sp, STACKFRAME_SIZE_ON_STACK
284
+
285
+
/* Switch to the per-CPU shadow call stack */
286
+
scs_save_current
287
+
scs_load_irq_stack t0
288
+
289
+
/* Switch to the per-CPU IRQ stack and call the handler */
290
+
load_per_cpu t0, irq_stack_ptr, t1
291
+
li t1, IRQ_STACK_SIZE
292
+
add sp, t0, t1
293
+
jalr a1
294
+
295
+
/* Switch back to the thread shadow call stack */
296
+
scs_load_current
297
+
298
+
/* Switch back to the thread stack and restore ra and s0 */
299
+
addi sp, s0, -STACKFRAME_SIZE_ON_STACK
300
+
REG_L ra, STACKFRAME_RA(sp)
301
+
REG_L s0, STACKFRAME_FP(sp)
302
+
addi sp, sp, STACKFRAME_SIZE_ON_STACK
303
+
304
+
ret
305
+
SYM_FUNC_END(call_on_irq_stack)
306
+
#endif /* CONFIG_IRQ_STACKS */
307
+
228
308
/*
229
309
* Integer register context switch
230
310
* The callee-saved registers must be saved and restored.
···
291
297
REG_S s9, TASK_THREAD_S9_RA(a3)
292
298
REG_S s10, TASK_THREAD_S10_RA(a3)
293
299
REG_S s11, TASK_THREAD_S11_RA(a3)
300
+
/* Save the kernel shadow call stack pointer */
301
+
scs_save_current
294
302
/* Restore context from next->thread */
295
303
REG_L ra, TASK_THREAD_RA_RA(a4)
296
304
REG_L sp, TASK_THREAD_SP_RA(a4)
···
310
314
REG_L s11, TASK_THREAD_S11_RA(a4)
311
315
/* The offset of thread_info in task_struct is zero. */
312
316
move tp, a1
317
+
/* Switch to the next shadow call stack */
318
+
scs_load_current
313
319
ret
314
320
SYM_FUNC_END(__switch_to)
315
321
+7
-12
arch/riscv/kernel/head.S
+7
-12
arch/riscv/kernel/head.S
···
14
14
#include <asm/cpu_ops_sbi.h>
15
15
#include <asm/hwcap.h>
16
16
#include <asm/image.h>
17
+
#include <asm/scs.h>
17
18
#include <asm/xip_fixup.h>
18
19
#include "efi-header.S"
19
20
···
111
110
csrw CSR_TVEC, a0
112
111
113
112
/* Reload the global pointer */
114
-
.option push
115
-
.option norelax
116
-
la gp, __global_pointer$
117
-
.option pop
113
+
load_global_pointer
118
114
119
115
/*
120
116
* Switch to kernel page tables. A full fence is necessary in order to
···
132
134
csrw CSR_IP, zero
133
135
134
136
/* Load the global pointer */
135
-
.option push
136
-
.option norelax
137
-
la gp, __global_pointer$
138
-
.option pop
137
+
load_global_pointer
139
138
140
139
/*
141
140
* Disable FPU & VECTOR to detect illegal usage of
···
154
159
XIP_FIXUP_OFFSET a3
155
160
add a3, a3, a1
156
161
REG_L sp, (a3)
162
+
scs_load_current
157
163
158
164
.Lsecondary_start_common:
159
165
···
224
228
#endif /* CONFIG_RISCV_M_MODE */
225
229
226
230
/* Load the global pointer */
227
-
.option push
228
-
.option norelax
229
-
la gp, __global_pointer$
230
-
.option pop
231
+
load_global_pointer
231
232
232
233
/*
233
234
* Disable FPU & VECTOR to detect illegal usage of
···
291
298
la sp, init_thread_union + THREAD_SIZE
292
299
XIP_FIXUP_OFFSET sp
293
300
addi sp, sp, -PT_SIZE_ON_STACK
301
+
scs_load_init_stack
294
302
#ifdef CONFIG_BUILTIN_DTB
295
303
la a0, __dtb_start
296
304
XIP_FIXUP_OFFSET a0
···
310
316
la tp, init_task
311
317
la sp, init_thread_union + THREAD_SIZE
312
318
addi sp, sp, -PT_SIZE_ON_STACK
319
+
scs_load_current
313
320
314
321
#ifdef CONFIG_KASAN
315
322
call kasan_early_init
+29
-27
arch/riscv/kernel/irq.c
+29
-27
arch/riscv/kernel/irq.c
···
9
9
#include <linux/irqchip.h>
10
10
#include <linux/irqdomain.h>
11
11
#include <linux/module.h>
12
+
#include <linux/scs.h>
12
13
#include <linux/seq_file.h>
13
14
#include <asm/sbi.h>
14
15
#include <asm/smp.h>
···
34
33
35
34
#ifdef CONFIG_IRQ_STACKS
36
35
#include <asm/irq_stack.h>
36
+
37
+
DECLARE_PER_CPU(ulong *, irq_shadow_call_stack_ptr);
38
+
39
+
#ifdef CONFIG_SHADOW_CALL_STACK
40
+
DEFINE_PER_CPU(ulong *, irq_shadow_call_stack_ptr);
41
+
#endif
42
+
43
+
static void init_irq_scs(void)
44
+
{
45
+
int cpu;
46
+
47
+
if (!scs_is_enabled())
48
+
return;
49
+
50
+
for_each_possible_cpu(cpu)
51
+
per_cpu(irq_shadow_call_stack_ptr, cpu) =
52
+
scs_alloc(cpu_to_node(cpu));
53
+
}
37
54
38
55
DEFINE_PER_CPU(ulong *, irq_stack_ptr);
39
56
···
80
61
#endif /* CONFIG_VMAP_STACK */
81
62
82
63
#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
64
+
static void ___do_softirq(struct pt_regs *regs)
65
+
{
66
+
__do_softirq();
67
+
}
68
+
83
69
void do_softirq_own_stack(void)
84
70
{
85
-
#ifdef CONFIG_IRQ_STACKS
86
-
if (on_thread_stack()) {
87
-
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
88
-
+ IRQ_STACK_SIZE/sizeof(ulong);
89
-
__asm__ __volatile(
90
-
"addi sp, sp, -"RISCV_SZPTR "\n"
91
-
REG_S" ra, (sp) \n"
92
-
"addi sp, sp, -"RISCV_SZPTR "\n"
93
-
REG_S" s0, (sp) \n"
94
-
"addi s0, sp, 2*"RISCV_SZPTR "\n"
95
-
"move sp, %[sp] \n"
96
-
"call __do_softirq \n"
97
-
"addi sp, s0, -2*"RISCV_SZPTR"\n"
98
-
REG_L" s0, (sp) \n"
99
-
"addi sp, sp, "RISCV_SZPTR "\n"
100
-
REG_L" ra, (sp) \n"
101
-
"addi sp, sp, "RISCV_SZPTR "\n"
102
-
:
103
-
: [sp] "r" (sp)
104
-
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
105
-
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
106
-
#ifndef CONFIG_FRAME_POINTER
107
-
"s0",
108
-
#endif
109
-
"memory");
110
-
} else
111
-
#endif
71
+
if (on_thread_stack())
72
+
call_on_irq_stack(NULL, ___do_softirq);
73
+
else
112
74
__do_softirq();
113
75
}
114
76
#endif /* CONFIG_SOFTIRQ_ON_OWN_STACK */
115
77
116
78
#else
79
+
static void init_irq_scs(void) {}
117
80
static void init_irq_stacks(void) {}
118
81
#endif /* CONFIG_IRQ_STACKS */
119
82
···
107
106
108
107
void __init init_IRQ(void)
109
108
{
109
+
init_irq_scs();
110
110
init_irq_stacks();
111
111
irqchip_init();
112
112
if (!handle_arch_irq)
+23
-29
arch/riscv/kernel/kexec_relocate.S
+23
-29
arch/riscv/kernel/kexec_relocate.S
···
17
17
* s1: (const) Phys address to jump to after relocation
18
18
* s2: (const) Phys address of the FDT image
19
19
* s3: (const) The hartid of the current hart
20
-
* s4: Pointer to the destination address for the relocation
21
-
* s5: (const) Number of words per page
22
-
* s6: (const) 1, used for subtraction
23
-
* s7: (const) kernel_map.va_pa_offset, used when switching MMU off
24
-
* s8: (const) Physical address of the main loop
25
-
* s9: (debug) indirection page counter
26
-
* s10: (debug) entry counter
27
-
* s11: (debug) copied words counter
20
+
* s4: (const) kernel_map.va_pa_offset, used when switching MMU off
21
+
* s5: Pointer to the destination address for the relocation
22
+
* s6: (const) Physical address of the main loop
28
23
*/
29
24
mv s0, a0
30
25
mv s1, a1
31
26
mv s2, a2
32
27
mv s3, a3
33
-
mv s4, zero
34
-
li s5, (PAGE_SIZE / RISCV_SZPTR)
35
-
li s6, 1
36
-
mv s7, a4
37
-
mv s8, zero
38
-
mv s9, zero
39
-
mv s10, zero
40
-
mv s11, zero
28
+
mv s4, a4
29
+
mv s5, zero
30
+
mv s6, zero
41
31
42
32
/* Disable / cleanup interrupts */
43
33
csrw CSR_SIE, zero
···
42
52
* the start of the loop below so that we jump there in
43
53
* any case.
44
54
*/
45
-
la s8, 1f
46
-
sub s8, s8, s7
47
-
csrw CSR_STVEC, s8
55
+
la s6, 1f
56
+
sub s6, s6, s4
57
+
csrw CSR_STVEC, s6
58
+
59
+
/*
60
+
* With C-extension, here we get 42 Bytes and the next
61
+
* .align directive would pad zeros here up to 44 Bytes.
62
+
* So manually put a nop here to avoid zeros padding.
63
+
*/
64
+
nop
48
65
49
66
/* Process entries in a loop */
50
67
.align 2
51
68
1:
52
-
addi s10, s10, 1
53
69
REG_L t0, 0(s0) /* t0 = *image->entry */
54
70
addi s0, s0, RISCV_SZPTR /* image->entry++ */
55
71
56
72
/* IND_DESTINATION entry ? -> save destination address */
57
73
andi t1, t0, 0x1
58
74
beqz t1, 2f
59
-
andi s4, t0, ~0x1
75
+
andi s5, t0, ~0x1
60
76
j 1b
61
77
62
78
2:
···
70
74
andi t1, t0, 0x2
71
75
beqz t1, 2f
72
76
andi s0, t0, ~0x2
73
-
addi s9, s9, 1
74
77
csrw CSR_SATP, zero
75
-
jalr zero, s8, 0
78
+
jr s6
76
79
77
80
2:
78
81
/* IND_DONE entry ? -> jump to done label */
···
87
92
andi t1, t0, 0x8
88
93
beqz t1, 1b /* Unknown entry type, ignore it */
89
94
andi t0, t0, ~0x8
90
-
mv t3, s5 /* i = num words per page */
95
+
li t3, (PAGE_SIZE / RISCV_SZPTR) /* i = num words per page */
91
96
3: /* copy loop */
92
97
REG_L t1, (t0) /* t1 = *src_ptr */
93
-
REG_S t1, (s4) /* *dst_ptr = *src_ptr */
98
+
REG_S t1, (s5) /* *dst_ptr = *src_ptr */
94
99
addi t0, t0, RISCV_SZPTR /* stc_ptr++ */
95
-
addi s4, s4, RISCV_SZPTR /* dst_ptr++ */
96
-
sub t3, t3, s6 /* i-- */
97
-
addi s11, s11, 1 /* c++ */
100
+
addi s5, s5, RISCV_SZPTR /* dst_ptr++ */
101
+
addi t3, t3, -0x1 /* i-- */
98
102
beqz t3, 1b /* copy done ? */
99
103
j 3b
100
104
···
140
146
*/
141
147
fence.i
142
148
143
-
jalr zero, a2, 0
149
+
jr a2
144
150
145
151
SYM_CODE_END(riscv_kexec_relocate)
146
152
riscv_kexec_relocate_end:
+4
arch/riscv/kernel/setup.c
+4
arch/riscv/kernel/setup.c
···
25
25
#include <asm/acpi.h>
26
26
#include <asm/alternative.h>
27
27
#include <asm/cacheflush.h>
28
+
#include <asm/cpufeature.h>
28
29
#include <asm/cpu_ops.h>
29
30
#include <asm/early_ioremap.h>
30
31
#include <asm/pgtable.h>
···
290
289
riscv_fill_hwcap();
291
290
init_rt_signal_env();
292
291
apply_boot_alternatives();
292
+
293
293
if (IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM) &&
294
294
riscv_isa_extension_available(NULL, ZICBOM))
295
295
riscv_noncoherent_supported();
296
296
riscv_set_dma_cache_alignment();
297
+
298
+
riscv_user_isa_enable();
297
299
}
298
300
299
301
static int __init topology_init(void)
+4
arch/riscv/kernel/smpboot.c
+4
arch/riscv/kernel/smpboot.c
···
25
25
#include <linux/of.h>
26
26
#include <linux/sched/task_stack.h>
27
27
#include <linux/sched/mm.h>
28
+
29
+
#include <asm/cpufeature.h>
28
30
#include <asm/cpu_ops.h>
29
31
#include <asm/cpufeature.h>
30
32
#include <asm/irq.h>
···
254
252
if (riscv_v_setup_vsize())
255
253
elf_hwcap &= ~COMPAT_HWCAP_ISA_V;
256
254
}
255
+
256
+
riscv_user_isa_enable();
257
257
258
258
/*
259
259
* Remote TLB flushes are ignored while the CPU is offline, so emit
+1
-4
arch/riscv/kernel/suspend_entry.S
+1
-4
arch/riscv/kernel/suspend_entry.S
+31
-13
arch/riscv/kernel/sys_riscv.c
+31
-13
arch/riscv/kernel/sys_riscv.c
···
145
145
for_each_cpu(cpu, cpus) {
146
146
struct riscv_isainfo *isainfo = &hart_isa[cpu];
147
147
148
-
if (riscv_isa_extension_available(isainfo->isa, ZBA))
149
-
pair->value |= RISCV_HWPROBE_EXT_ZBA;
150
-
else
151
-
missing |= RISCV_HWPROBE_EXT_ZBA;
148
+
#define EXT_KEY(ext) \
149
+
do { \
150
+
if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \
151
+
pair->value |= RISCV_HWPROBE_EXT_##ext; \
152
+
else \
153
+
missing |= RISCV_HWPROBE_EXT_##ext; \
154
+
} while (false)
152
155
153
-
if (riscv_isa_extension_available(isainfo->isa, ZBB))
154
-
pair->value |= RISCV_HWPROBE_EXT_ZBB;
155
-
else
156
-
missing |= RISCV_HWPROBE_EXT_ZBB;
157
-
158
-
if (riscv_isa_extension_available(isainfo->isa, ZBS))
159
-
pair->value |= RISCV_HWPROBE_EXT_ZBS;
160
-
else
161
-
missing |= RISCV_HWPROBE_EXT_ZBS;
156
+
/*
157
+
* Only use EXT_KEY() for extensions which can be exposed to userspace,
158
+
* regardless of the kernel's configuration, as no other checks, besides
159
+
* presence in the hart_isa bitmap, are made.
160
+
*/
161
+
EXT_KEY(ZBA);
162
+
EXT_KEY(ZBB);
163
+
EXT_KEY(ZBS);
164
+
EXT_KEY(ZICBOZ);
165
+
#undef EXT_KEY
162
166
}
163
167
164
168
/* Now turn off reporting features if any CPU is missing it. */
165
169
pair->value &= ~missing;
170
+
}
171
+
172
+
static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext)
173
+
{
174
+
struct riscv_hwprobe pair;
175
+
176
+
hwprobe_isa_ext0(&pair, cpus);
177
+
return (pair.value & ext);
166
178
}
167
179
168
180
static u64 hwprobe_misaligned(const struct cpumask *cpus)
···
225
213
226
214
case RISCV_HWPROBE_KEY_CPUPERF_0:
227
215
pair->value = hwprobe_misaligned(cpus);
216
+
break;
217
+
218
+
case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
219
+
pair->value = 0;
220
+
if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
221
+
pair->value = riscv_cboz_block_size;
228
222
break;
229
223
230
224
/*
+5
-63
arch/riscv/kernel/traps.c
+5
-63
arch/riscv/kernel/traps.c
···
360
360
asmlinkage void noinstr do_irq(struct pt_regs *regs)
361
361
{
362
362
irqentry_state_t state = irqentry_enter(regs);
363
-
#ifdef CONFIG_IRQ_STACKS
364
-
if (on_thread_stack()) {
365
-
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
366
-
+ IRQ_STACK_SIZE/sizeof(ulong);
367
-
__asm__ __volatile(
368
-
"addi sp, sp, -"RISCV_SZPTR "\n"
369
-
REG_S" ra, (sp) \n"
370
-
"addi sp, sp, -"RISCV_SZPTR "\n"
371
-
REG_S" s0, (sp) \n"
372
-
"addi s0, sp, 2*"RISCV_SZPTR "\n"
373
-
"move sp, %[sp] \n"
374
-
"move a0, %[regs] \n"
375
-
"call handle_riscv_irq \n"
376
-
"addi sp, s0, -2*"RISCV_SZPTR"\n"
377
-
REG_L" s0, (sp) \n"
378
-
"addi sp, sp, "RISCV_SZPTR "\n"
379
-
REG_L" ra, (sp) \n"
380
-
"addi sp, sp, "RISCV_SZPTR "\n"
381
-
:
382
-
: [sp] "r" (sp), [regs] "r" (regs)
383
-
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
384
-
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
385
-
#ifndef CONFIG_FRAME_POINTER
386
-
"s0",
387
-
#endif
388
-
"memory");
389
-
} else
390
-
#endif
363
+
364
+
if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
365
+
call_on_irq_stack(regs, handle_riscv_irq);
366
+
else
391
367
handle_riscv_irq(regs);
392
368
393
369
irqentry_exit(regs, state);
···
386
410
#endif /* CONFIG_GENERIC_BUG */
387
411
388
412
#ifdef CONFIG_VMAP_STACK
389
-
/*
390
-
* Extra stack space that allows us to provide panic messages when the kernel
391
-
* has overflowed its stack.
392
-
*/
393
-
static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
413
+
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
394
414
overflow_stack)__aligned(16);
395
-
/*
396
-
* A temporary stack for use by handle_kernel_stack_overflow. This is used so
397
-
* we can call into C code to get the per-hart overflow stack. Usage of this
398
-
* stack must be protected by spin_shadow_stack.
399
-
*/
400
-
long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
401
-
402
-
/*
403
-
* A pseudo spinlock to protect the shadow stack from being used by multiple
404
-
* harts concurrently. This isn't a real spinlock because the lock side must
405
-
* be taken without a valid stack and only a single register, it's only taken
406
-
* while in the process of panicing anyway so the performance and error
407
-
* checking a proper spinlock gives us doesn't matter.
408
-
*/
409
-
unsigned long spin_shadow_stack;
410
-
411
-
asmlinkage unsigned long get_overflow_stack(void)
412
-
{
413
-
return (unsigned long)this_cpu_ptr(overflow_stack) +
414
-
OVERFLOW_STACK_SIZE;
415
-
}
416
415
417
416
asmlinkage void handle_bad_stack(struct pt_regs *regs)
418
417
{
419
418
unsigned long tsk_stk = (unsigned long)current->stack;
420
419
unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
421
-
422
-
/*
423
-
* We're done with the shadow stack by this point, as we're on the
424
-
* overflow stack. Tell any other concurrent overflowing harts that
425
-
* they can proceed with panicing by releasing the pseudo-spinlock.
426
-
*
427
-
* This pairs with an amoswap.aq in handle_kernel_stack_overflow.
428
-
*/
429
-
smp_store_release(&spin_shadow_stack, 0);
430
420
431
421
console_verbose();
432
422
+1
-1
arch/riscv/kernel/vdso/Makefile
+1
-1
arch/riscv/kernel/vdso/Makefile
+1
-1
arch/riscv/kernel/vdso/hwprobe.c
+1
-1
arch/riscv/kernel/vdso/hwprobe.c
+19
-6
arch/riscv/mm/cacheflush.c
+19
-6
arch/riscv/mm/cacheflush.c
···
3
3
* Copyright (C) 2017 SiFive
4
4
*/
5
5
6
+
#include <linux/acpi.h>
6
7
#include <linux/of.h>
8
+
#include <asm/acpi.h>
7
9
#include <asm/cacheflush.h>
8
10
9
11
#ifdef CONFIG_SMP
···
126
124
unsigned long cbom_hartid, cboz_hartid;
127
125
u32 cbom_block_size = 0, cboz_block_size = 0;
128
126
struct device_node *node;
127
+
struct acpi_table_header *rhct;
128
+
acpi_status status;
129
129
130
-
for_each_of_cpu_node(node) {
131
-
/* set block-size for cbom and/or cboz extension if available */
132
-
cbo_get_block_size(node, "riscv,cbom-block-size",
133
-
&cbom_block_size, &cbom_hartid);
134
-
cbo_get_block_size(node, "riscv,cboz-block-size",
135
-
&cboz_block_size, &cboz_hartid);
130
+
if (acpi_disabled) {
131
+
for_each_of_cpu_node(node) {
132
+
/* set block-size for cbom and/or cboz extension if available */
133
+
cbo_get_block_size(node, "riscv,cbom-block-size",
134
+
&cbom_block_size, &cbom_hartid);
135
+
cbo_get_block_size(node, "riscv,cboz-block-size",
136
+
&cboz_block_size, &cboz_hartid);
137
+
}
138
+
} else {
139
+
status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
140
+
if (ACPI_FAILURE(status))
141
+
return;
142
+
143
+
acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL);
144
+
acpi_put_table((struct acpi_table_header *)rhct);
136
145
}
137
146
138
147
if (cbom_block_size)
+2
arch/riscv/mm/init.c
+2
arch/riscv/mm/init.c
···
49
49
#endif
50
50
EXPORT_SYMBOL(satp_mode);
51
51
52
+
#ifdef CONFIG_64BIT
52
53
bool pgtable_l4_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL);
53
54
bool pgtable_l5_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL);
54
55
EXPORT_SYMBOL(pgtable_l4_enabled);
55
56
EXPORT_SYMBOL(pgtable_l5_enabled);
57
+
#endif
56
58
57
59
phys_addr_t phys_ram_base __ro_after_init;
58
60
EXPORT_SYMBOL(phys_ram_base);
+3
arch/riscv/mm/ptdump.c
+3
arch/riscv/mm/ptdump.c
···
384
384
385
385
kernel_ptd_info.base_addr = KERN_VIRT_START;
386
386
387
+
pg_level[1].name = pgtable_l5_enabled ? "P4D" : "PGD";
388
+
pg_level[2].name = pgtable_l4_enabled ? "PUD" : "PGD";
389
+
387
390
for (i = 0; i < ARRAY_SIZE(pg_level); i++)
388
391
for (j = 0; j < ARRAY_SIZE(pte_bits); j++)
389
392
pg_level[i].mask |= pte_bits[j].mask;
+8
arch/riscv/purgatory/Makefile
+8
arch/riscv/purgatory/Makefile
···
81
81
PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI)
82
82
endif
83
83
84
+
ifdef CONFIG_RELOCATABLE
85
+
PURGATORY_CFLAGS_REMOVE += -fPIE
86
+
endif
87
+
88
+
ifdef CONFIG_SHADOW_CALL_STACK
89
+
PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_SCS)
90
+
endif
91
+
84
92
CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE)
85
93
CFLAGS_purgatory.o += $(PURGATORY_CFLAGS)
86
94
+90
-3
drivers/acpi/riscv/rhct.c
+90
-3
drivers/acpi/riscv/rhct.c
···
8
8
#define pr_fmt(fmt) "ACPI: RHCT: " fmt
9
9
10
10
#include <linux/acpi.h>
11
+
#include <linux/bits.h>
11
12
12
-
static struct acpi_table_header *acpi_get_rhct(void)
13
+
static struct acpi_table_rhct *acpi_get_rhct(void)
13
14
{
14
15
static struct acpi_table_header *rhct;
15
16
acpi_status status;
···
27
26
}
28
27
}
29
28
30
-
return rhct;
29
+
return (struct acpi_table_rhct *)rhct;
31
30
}
32
31
33
32
/*
···
49
48
BUG_ON(acpi_disabled);
50
49
51
50
if (!table) {
52
-
rhct = (struct acpi_table_rhct *)acpi_get_rhct();
51
+
rhct = acpi_get_rhct();
53
52
if (!rhct)
54
53
return -ENOENT;
55
54
} else {
···
81
80
}
82
81
83
82
return -1;
83
+
}
84
+
85
+
static void acpi_parse_hart_info_cmo_node(struct acpi_table_rhct *rhct,
86
+
struct acpi_rhct_hart_info *hart_info,
87
+
u32 *cbom_size, u32 *cboz_size, u32 *cbop_size)
88
+
{
89
+
u32 size_hartinfo = sizeof(struct acpi_rhct_hart_info);
90
+
u32 size_hdr = sizeof(struct acpi_rhct_node_header);
91
+
struct acpi_rhct_node_header *ref_node;
92
+
struct acpi_rhct_cmo_node *cmo_node;
93
+
u32 *hart_info_node_offset;
94
+
95
+
hart_info_node_offset = ACPI_ADD_PTR(u32, hart_info, size_hartinfo);
96
+
for (int i = 0; i < hart_info->num_offsets; i++) {
97
+
ref_node = ACPI_ADD_PTR(struct acpi_rhct_node_header,
98
+
rhct, hart_info_node_offset[i]);
99
+
if (ref_node->type == ACPI_RHCT_NODE_TYPE_CMO) {
100
+
cmo_node = ACPI_ADD_PTR(struct acpi_rhct_cmo_node,
101
+
ref_node, size_hdr);
102
+
if (cbom_size && cmo_node->cbom_size <= 30) {
103
+
if (!*cbom_size)
104
+
*cbom_size = BIT(cmo_node->cbom_size);
105
+
else if (*cbom_size != BIT(cmo_node->cbom_size))
106
+
pr_warn("CBOM size is not the same across harts\n");
107
+
}
108
+
109
+
if (cboz_size && cmo_node->cboz_size <= 30) {
110
+
if (!*cboz_size)
111
+
*cboz_size = BIT(cmo_node->cboz_size);
112
+
else if (*cboz_size != BIT(cmo_node->cboz_size))
113
+
pr_warn("CBOZ size is not the same across harts\n");
114
+
}
115
+
116
+
if (cbop_size && cmo_node->cbop_size <= 30) {
117
+
if (!*cbop_size)
118
+
*cbop_size = BIT(cmo_node->cbop_size);
119
+
else if (*cbop_size != BIT(cmo_node->cbop_size))
120
+
pr_warn("CBOP size is not the same across harts\n");
121
+
}
122
+
}
123
+
}
124
+
}
125
+
126
+
/*
127
+
* During early boot, the caller should call acpi_get_table() and pass its pointer to
128
+
* these functions (and free up later). At run time, since this table can be used
129
+
* multiple times, pass NULL so that the table remains in memory.
130
+
*/
131
+
void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size,
132
+
u32 *cboz_size, u32 *cbop_size)
133
+
{
134
+
u32 size_hdr = sizeof(struct acpi_rhct_node_header);
135
+
struct acpi_rhct_node_header *node, *end;
136
+
struct acpi_rhct_hart_info *hart_info;
137
+
struct acpi_table_rhct *rhct;
138
+
139
+
if (acpi_disabled)
140
+
return;
141
+
142
+
if (table) {
143
+
rhct = (struct acpi_table_rhct *)table;
144
+
} else {
145
+
rhct = acpi_get_rhct();
146
+
if (!rhct)
147
+
return;
148
+
}
149
+
150
+
if (cbom_size)
151
+
*cbom_size = 0;
152
+
153
+
if (cboz_size)
154
+
*cboz_size = 0;
155
+
156
+
if (cbop_size)
157
+
*cbop_size = 0;
158
+
159
+
end = ACPI_ADD_PTR(struct acpi_rhct_node_header, rhct, rhct->header.length);
160
+
for (node = ACPI_ADD_PTR(struct acpi_rhct_node_header, rhct, rhct->node_offset);
161
+
node < end;
162
+
node = ACPI_ADD_PTR(struct acpi_rhct_node_header, node, node->length)) {
163
+
if (node->type == ACPI_RHCT_NODE_TYPE_HART_INFO) {
164
+
hart_info = ACPI_ADD_PTR(struct acpi_rhct_hart_info, node, size_hdr);
165
+
acpi_parse_hart_info_cmo_node(rhct, hart_info, cbom_size,
166
+
cboz_size, cbop_size);
167
+
}
168
+
}
84
169
}
+15
-2
drivers/clocksource/timer-riscv.c
+15
-2
drivers/clocksource/timer-riscv.c
···
22
22
#include <linux/io-64-nonatomic-lo-hi.h>
23
23
#include <linux/interrupt.h>
24
24
#include <linux/of_irq.h>
25
+
#include <linux/limits.h>
25
26
#include <clocksource/timer-riscv.h>
26
27
#include <asm/smp.h>
27
28
#include <asm/hwcap.h>
···
32
31
static DEFINE_STATIC_KEY_FALSE(riscv_sstc_available);
33
32
static bool riscv_timer_cannot_wake_cpu;
34
33
34
+
static void riscv_clock_event_stop(void)
35
+
{
36
+
if (static_branch_likely(&riscv_sstc_available)) {
37
+
csr_write(CSR_STIMECMP, ULONG_MAX);
38
+
if (IS_ENABLED(CONFIG_32BIT))
39
+
csr_write(CSR_STIMECMPH, ULONG_MAX);
40
+
} else {
41
+
sbi_set_timer(U64_MAX);
42
+
}
43
+
}
44
+
35
45
static int riscv_clock_next_event(unsigned long delta,
36
46
struct clock_event_device *ce)
37
47
{
38
48
u64 next_tval = get_cycles64() + delta;
39
49
40
-
csr_set(CSR_IE, IE_TIE);
41
50
if (static_branch_likely(&riscv_sstc_available)) {
42
51
#if defined(CONFIG_32BIT)
43
52
csr_write(CSR_STIMECMP, next_tval & 0xFFFFFFFF);
···
105
94
ce->irq = riscv_clock_event_irq;
106
95
if (riscv_timer_cannot_wake_cpu)
107
96
ce->features |= CLOCK_EVT_FEAT_C3STOP;
97
+
if (static_branch_likely(&riscv_sstc_available))
98
+
ce->rating = 450;
108
99
clockevents_config_and_register(ce, riscv_timebase, 100, 0x7fffffff);
109
100
110
101
enable_percpu_irq(riscv_clock_event_irq,
···
132
119
{
133
120
struct clock_event_device *evdev = this_cpu_ptr(&riscv_clock_event);
134
121
135
-
csr_clear(CSR_IE, IE_TIE);
122
+
riscv_clock_event_stop();
136
123
evdev->event_handler(evdev);
137
124
138
125
return IRQ_HANDLED;
+11
-2
drivers/misc/lkdtm/cfi.c
+11
-2
drivers/misc/lkdtm/cfi.c
···
68
68
#define no_pac_addr(addr) \
69
69
((__force __typeof__(addr))((uintptr_t)(addr) | PAGE_OFFSET))
70
70
71
+
#ifdef CONFIG_RISCV
72
+
/* https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#frame-pointer-convention */
73
+
#define FRAME_RA_OFFSET (-1)
74
+
#else
75
+
#define FRAME_RA_OFFSET 1
76
+
#endif
77
+
71
78
/* The ultimate ROP gadget. */
72
79
static noinline __no_ret_protection
73
80
void set_return_addr_unchecked(unsigned long *expected, unsigned long *addr)
74
81
{
75
82
/* Use of volatile is to make sure final write isn't seen as a dead store. */
76
-
unsigned long * volatile *ret_addr = (unsigned long **)__builtin_frame_address(0) + 1;
83
+
unsigned long * volatile *ret_addr =
84
+
(unsigned long **)__builtin_frame_address(0) + FRAME_RA_OFFSET;
77
85
78
86
/* Make sure we've found the right place on the stack before writing it. */
79
87
if (no_pac_addr(*ret_addr) == expected)
···
96
88
void set_return_addr(unsigned long *expected, unsigned long *addr)
97
89
{
98
90
/* Use of volatile is to make sure final write isn't seen as a dead store. */
99
-
unsigned long * volatile *ret_addr = (unsigned long **)__builtin_frame_address(0) + 1;
91
+
unsigned long * volatile *ret_addr =
92
+
(unsigned long **)__builtin_frame_address(0) + FRAME_RA_OFFSET;
100
93
101
94
/* Make sure we've found the right place on the stack before writing it. */
102
95
if (no_pac_addr(*ret_addr) == expected)
+7
-2
tools/testing/selftests/riscv/hwprobe/Makefile
+7
-2
tools/testing/selftests/riscv/hwprobe/Makefile
···
2
2
# Copyright (C) 2021 ARM Limited
3
3
# Originally tools/testing/arm64/abi/Makefile
4
4
5
-
TEST_GEN_PROGS := hwprobe
5
+
CFLAGS += -I$(top_srcdir)/tools/include
6
+
7
+
TEST_GEN_PROGS := hwprobe cbo
6
8
7
9
include ../../lib.mk
8
10
9
11
$(OUTPUT)/hwprobe: hwprobe.c sys_hwprobe.S
10
-
$(CC) -o$@ $(CFLAGS) $(LDFLAGS) $^
12
+
$(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^
13
+
14
+
$(OUTPUT)/cbo: cbo.c sys_hwprobe.S
15
+
$(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^
+228
tools/testing/selftests/riscv/hwprobe/cbo.c
+228
tools/testing/selftests/riscv/hwprobe/cbo.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (c) 2023 Ventana Micro Systems Inc.
4
+
*
5
+
* Run with 'taskset -c <cpu-list> cbo' to only execute hwprobe on a
6
+
* subset of cpus, as well as only executing the tests on those cpus.
7
+
*/
8
+
#define _GNU_SOURCE
9
+
#include <stdbool.h>
10
+
#include <stdint.h>
11
+
#include <string.h>
12
+
#include <sched.h>
13
+
#include <signal.h>
14
+
#include <assert.h>
15
+
#include <linux/compiler.h>
16
+
#include <linux/kernel.h>
17
+
#include <asm/ucontext.h>
18
+
19
+
#include "hwprobe.h"
20
+
#include "../../kselftest.h"
21
+
22
+
#define MK_CBO(fn) cpu_to_le32((fn) << 20 | 10 << 15 | 2 << 12 | 0 << 7 | 15)
23
+
24
+
static char mem[4096] __aligned(4096) = { [0 ... 4095] = 0xa5 };
25
+
26
+
static bool illegal_insn;
27
+
28
+
static void sigill_handler(int sig, siginfo_t *info, void *context)
29
+
{
30
+
unsigned long *regs = (unsigned long *)&((ucontext_t *)context)->uc_mcontext;
31
+
uint32_t insn = *(uint32_t *)regs[0];
32
+
33
+
assert(insn == MK_CBO(regs[11]));
34
+
35
+
illegal_insn = true;
36
+
regs[0] += 4;
37
+
}
38
+
39
+
static void cbo_insn(char *base, int fn)
40
+
{
41
+
uint32_t insn = MK_CBO(fn);
42
+
43
+
asm volatile(
44
+
"mv a0, %0\n"
45
+
"li a1, %1\n"
46
+
".4byte %2\n"
47
+
: : "r" (base), "i" (fn), "i" (insn) : "a0", "a1", "memory");
48
+
}
49
+
50
+
static void cbo_inval(char *base) { cbo_insn(base, 0); }
51
+
static void cbo_clean(char *base) { cbo_insn(base, 1); }
52
+
static void cbo_flush(char *base) { cbo_insn(base, 2); }
53
+
static void cbo_zero(char *base) { cbo_insn(base, 4); }
54
+
55
+
static void test_no_zicbom(void *arg)
56
+
{
57
+
ksft_print_msg("Testing Zicbom instructions remain privileged\n");
58
+
59
+
illegal_insn = false;
60
+
cbo_clean(&mem[0]);
61
+
ksft_test_result(illegal_insn, "No cbo.clean\n");
62
+
63
+
illegal_insn = false;
64
+
cbo_flush(&mem[0]);
65
+
ksft_test_result(illegal_insn, "No cbo.flush\n");
66
+
67
+
illegal_insn = false;
68
+
cbo_inval(&mem[0]);
69
+
ksft_test_result(illegal_insn, "No cbo.inval\n");
70
+
}
71
+
72
+
static void test_no_zicboz(void *arg)
73
+
{
74
+
ksft_print_msg("No Zicboz, testing cbo.zero remains privileged\n");
75
+
76
+
illegal_insn = false;
77
+
cbo_zero(&mem[0]);
78
+
ksft_test_result(illegal_insn, "No cbo.zero\n");
79
+
}
80
+
81
+
static bool is_power_of_2(__u64 n)
82
+
{
83
+
return n != 0 && (n & (n - 1)) == 0;
84
+
}
85
+
86
+
static void test_zicboz(void *arg)
87
+
{
88
+
struct riscv_hwprobe pair = {
89
+
.key = RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE,
90
+
};
91
+
cpu_set_t *cpus = (cpu_set_t *)arg;
92
+
__u64 block_size;
93
+
int i, j;
94
+
long rc;
95
+
96
+
rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
97
+
block_size = pair.value;
98
+
ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE &&
99
+
is_power_of_2(block_size), "Zicboz block size\n");
100
+
ksft_print_msg("Zicboz block size: %ld\n", block_size);
101
+
102
+
illegal_insn = false;
103
+
cbo_zero(&mem[block_size]);
104
+
ksft_test_result(!illegal_insn, "cbo.zero\n");
105
+
106
+
if (illegal_insn || !is_power_of_2(block_size)) {
107
+
ksft_test_result_skip("cbo.zero check\n");
108
+
return;
109
+
}
110
+
111
+
assert(block_size <= 1024);
112
+
113
+
for (i = 0; i < 4096 / block_size; ++i) {
114
+
if (i % 2)
115
+
cbo_zero(&mem[i * block_size]);
116
+
}
117
+
118
+
for (i = 0; i < 4096 / block_size; ++i) {
119
+
char expected = i % 2 ? 0x0 : 0xa5;
120
+
121
+
for (j = 0; j < block_size; ++j) {
122
+
if (mem[i * block_size + j] != expected) {
123
+
ksft_test_result_fail("cbo.zero check\n");
124
+
ksft_print_msg("cbo.zero check: mem[%d] != 0x%x\n",
125
+
i * block_size + j, expected);
126
+
return;
127
+
}
128
+
}
129
+
}
130
+
131
+
ksft_test_result_pass("cbo.zero check\n");
132
+
}
133
+
134
+
static void check_no_zicboz_cpus(cpu_set_t *cpus)
135
+
{
136
+
struct riscv_hwprobe pair = {
137
+
.key = RISCV_HWPROBE_KEY_IMA_EXT_0,
138
+
};
139
+
cpu_set_t one_cpu;
140
+
int i = 0, c = 0;
141
+
long rc;
142
+
143
+
while (i++ < CPU_COUNT(cpus)) {
144
+
while (!CPU_ISSET(c, cpus))
145
+
++c;
146
+
147
+
CPU_ZERO(&one_cpu);
148
+
CPU_SET(c, &one_cpu);
149
+
150
+
rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0);
151
+
assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
152
+
153
+
if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ)
154
+
ksft_exit_fail_msg("Zicboz is only present on a subset of harts.\n"
155
+
"Use taskset to select a set of harts where Zicboz\n"
156
+
"presence (present or not) is consistent for each hart\n");
157
+
++c;
158
+
}
159
+
}
160
+
161
+
enum {
162
+
TEST_ZICBOZ,
163
+
TEST_NO_ZICBOZ,
164
+
TEST_NO_ZICBOM,
165
+
};
166
+
167
+
static struct test_info {
168
+
bool enabled;
169
+
unsigned int nr_tests;
170
+
void (*test_fn)(void *arg);
171
+
} tests[] = {
172
+
[TEST_ZICBOZ] = { .nr_tests = 3, test_zicboz },
173
+
[TEST_NO_ZICBOZ] = { .nr_tests = 1, test_no_zicboz },
174
+
[TEST_NO_ZICBOM] = { .nr_tests = 3, test_no_zicbom },
175
+
};
176
+
177
+
int main(int argc, char **argv)
178
+
{
179
+
struct sigaction act = {
180
+
.sa_sigaction = &sigill_handler,
181
+
.sa_flags = SA_SIGINFO,
182
+
};
183
+
struct riscv_hwprobe pair;
184
+
unsigned int plan = 0;
185
+
cpu_set_t cpus;
186
+
long rc;
187
+
int i;
188
+
189
+
if (argc > 1 && !strcmp(argv[1], "--sigill")) {
190
+
rc = sigaction(SIGILL, &act, NULL);
191
+
assert(rc == 0);
192
+
tests[TEST_NO_ZICBOZ].enabled = true;
193
+
tests[TEST_NO_ZICBOM].enabled = true;
194
+
}
195
+
196
+
rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus);
197
+
assert(rc == 0);
198
+
199
+
ksft_print_header();
200
+
201
+
pair.key = RISCV_HWPROBE_KEY_IMA_EXT_0;
202
+
rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&cpus, 0);
203
+
if (rc < 0)
204
+
ksft_exit_fail_msg("hwprobe() failed with %d\n", rc);
205
+
assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
206
+
207
+
if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ) {
208
+
tests[TEST_ZICBOZ].enabled = true;
209
+
tests[TEST_NO_ZICBOZ].enabled = false;
210
+
} else {
211
+
check_no_zicboz_cpus(&cpus);
212
+
}
213
+
214
+
for (i = 0; i < ARRAY_SIZE(tests); ++i)
215
+
plan += tests[i].enabled ? tests[i].nr_tests : 0;
216
+
217
+
if (plan == 0)
218
+
ksft_print_msg("No tests enabled.\n");
219
+
else
220
+
ksft_set_plan(plan);
221
+
222
+
for (i = 0; i < ARRAY_SIZE(tests); ++i) {
223
+
if (tests[i].enabled)
224
+
tests[i].test_fn(&cpus);
225
+
}
226
+
227
+
ksft_finished();
228
+
}
+20
-44
tools/testing/selftests/riscv/hwprobe/hwprobe.c
+20
-44
tools/testing/selftests/riscv/hwprobe/hwprobe.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
-
#include <stddef.h>
3
-
#include <asm/hwprobe.h>
4
-
5
-
/*
6
-
* Rather than relying on having a new enough libc to define this, just do it
7
-
* ourselves. This way we don't need to be coupled to a new-enough libc to
8
-
* contain the call.
9
-
*/
10
-
long riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
11
-
size_t cpu_count, unsigned long *cpus, unsigned int flags);
2
+
#include "hwprobe.h"
3
+
#include "../../kselftest.h"
12
4
13
5
int main(int argc, char **argv)
14
6
{
15
7
struct riscv_hwprobe pairs[8];
16
8
unsigned long cpus;
17
9
long out;
10
+
11
+
ksft_print_header();
12
+
ksft_set_plan(5);
18
13
19
14
/* Fake the CPU_SET ops. */
20
15
cpus = -1;
···
20
25
*/
21
26
for (long i = 0; i < 8; i++)
22
27
pairs[i].key = i;
28
+
23
29
out = riscv_hwprobe(pairs, 8, 1, &cpus, 0);
24
30
if (out != 0)
25
-
return -1;
31
+
ksft_exit_fail_msg("hwprobe() failed with %ld\n", out);
32
+
26
33
for (long i = 0; i < 4; ++i) {
27
34
/* Fail if the kernel claims not to recognize a base key. */
28
35
if ((i < 4) && (pairs[i].key != i))
29
-
return -2;
36
+
ksft_exit_fail_msg("Failed to recognize base key: key != i, "
37
+
"key=%ld, i=%ld\n", pairs[i].key, i);
30
38
31
39
if (pairs[i].key != RISCV_HWPROBE_KEY_BASE_BEHAVIOR)
32
40
continue;
···
37
39
if (pairs[i].value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA)
38
40
continue;
39
41
40
-
return -3;
42
+
ksft_exit_fail_msg("Unexpected pair: (%ld, %ld)\n", pairs[i].key, pairs[i].value);
41
43
}
42
44
43
-
/*
44
-
* This should also work with a NULL CPU set, but should not work
45
-
* with an improperly supplied CPU set.
46
-
*/
47
45
out = riscv_hwprobe(pairs, 8, 0, 0, 0);
48
-
if (out != 0)
49
-
return -4;
46
+
ksft_test_result(out == 0, "NULL CPU set\n");
50
47
51
48
out = riscv_hwprobe(pairs, 8, 0, &cpus, 0);
52
-
if (out == 0)
53
-
return -5;
49
+
ksft_test_result(out != 0, "Bad CPU set\n");
54
50
55
51
out = riscv_hwprobe(pairs, 8, 1, 0, 0);
56
-
if (out == 0)
57
-
return -6;
52
+
ksft_test_result(out != 0, "NULL CPU set with non-zero count\n");
58
53
59
-
/*
60
-
* Check that keys work by providing one that we know exists, and
61
-
* checking to make sure the resultig pair is what we asked for.
62
-
*/
63
54
pairs[0].key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR;
64
55
out = riscv_hwprobe(pairs, 1, 1, &cpus, 0);
65
-
if (out != 0)
66
-
return -7;
67
-
if (pairs[0].key != RISCV_HWPROBE_KEY_BASE_BEHAVIOR)
68
-
return -8;
56
+
ksft_test_result(out == 0 && pairs[0].key == RISCV_HWPROBE_KEY_BASE_BEHAVIOR,
57
+
"Existing key is maintained\n");
69
58
70
-
/*
71
-
* Check that an unknown key gets overwritten with -1,
72
-
* but doesn't block elements after it.
73
-
*/
74
59
pairs[0].key = 0x5555;
75
60
pairs[1].key = 1;
76
61
pairs[1].value = 0xAAAA;
77
62
out = riscv_hwprobe(pairs, 2, 0, 0, 0);
78
-
if (out != 0)
79
-
return -9;
63
+
ksft_test_result(out == 0 && pairs[0].key == -1 &&
64
+
pairs[1].key == 1 && pairs[1].value != 0xAAAA,
65
+
"Unknown key overwritten with -1 and doesn't block other elements\n");
80
66
81
-
if (pairs[0].key != -1)
82
-
return -10;
83
-
84
-
if ((pairs[1].key != 1) || (pairs[1].value == 0xAAAA))
85
-
return -11;
86
-
87
-
return 0;
67
+
ksft_finished();
88
68
}
+15
tools/testing/selftests/riscv/hwprobe/hwprobe.h
+15
tools/testing/selftests/riscv/hwprobe/hwprobe.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
#ifndef SELFTEST_RISCV_HWPROBE_H
3
+
#define SELFTEST_RISCV_HWPROBE_H
4
+
#include <stddef.h>
5
+
#include <asm/hwprobe.h>
6
+
7
+
/*
8
+
* Rather than relying on having a new enough libc to define this, just do it
9
+
* ourselves. This way we don't need to be coupled to a new-enough libc to
10
+
* contain the call.
11
+
*/
12
+
long riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
13
+
size_t cpu_count, unsigned long *cpus, unsigned int flags);
14
+
15
+
#endif