powerpc/64s/radix/kfence: map __kfence_pool at page granularity

When KFENCE is enabled, total system memory is mapped at page level
granularity. But in radix MMU mode, ~3GB additional memory is needed
to map 100GB of system memory at page level granularity when compared
to using 2MB direct mapping.This is not desired considering KFENCE is
designed to be enabled in production kernels [1].

Mapping only the memory allocated for KFENCE pool at page granularity is
sufficient to enable KFENCE support. So, allocate __kfence_pool during
bootup and map it at page granularity instead of mapping all system
memory at page granularity.

Without patch:
# cat /proc/meminfo
MemTotal: 101201920 kB

With patch:
# cat /proc/meminfo
MemTotal: 104483904 kB

Note that enabling KFENCE at runtime is disabled for radix MMU for now,
as it depends on the ability to split page table mappings and such APIs
are not currently implemented for radix MMU.

All kfence_test.c testcases passed with this patch.

[1] https://lore.kernel.org/all/20201103175841.3495947-2-elver@google.com/

Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://msgid.link/20240701130021.578240-1-hbathini@linux.ibm.com

authored by

Hari Bathini and committed by

Michael Ellerman 2 years ago 353d7a84 af199e6c

+93 -5

3 changed files

expand all

arch

powerpc

include

asm

kfence.h

book3s64

radix_pgtable.c

init-common.c

+10 -1

arch/powerpc/include/asm/kfence.h

··· 15 15 #define ARCH_FUNC_PREFIX "." 16 16 #endif 17 17 18 + #ifdef CONFIG_KFENCE 19 + extern bool kfence_disabled; 20 + 21 + static inline void disable_kfence(void) 22 + { 23 + kfence_disabled = true; 24 + } 25 + 18 26 static inline bool arch_kfence_init_pool(void) 19 27 { 20 - return true; 28 + return !kfence_disabled; 21 29 } 30 + #endif 22 31 23 32 #ifdef CONFIG_PPC64 24 33 static inline bool kfence_protect_page(unsigned long addr, bool protect)

+80 -4

arch/powerpc/mm/book3s64/radix_pgtable.c

··· 17 17 #include <linux/hugetlb.h> 18 18 #include <linux/string_helpers.h> 19 19 #include <linux/memory.h> 20 + #include <linux/kfence.h> 20 21 21 22 #include <asm/pgalloc.h> 22 23 #include <asm/mmu_context.h> ··· 32 31 #include <asm/uaccess.h> 33 32 #include <asm/ultravisor.h> 34 33 #include <asm/set_memory.h> 34 + #include <asm/kfence.h> 35 35 36 36 #include <trace/events/thp.h> 37 37 ··· 295 293 296 294 static int __meminit create_physical_mapping(unsigned long start, 297 295 unsigned long end, 298 - int nid, pgprot_t _prot) 296 + int nid, pgprot_t _prot, 297 + unsigned long mapping_sz_limit) 299 298 { 300 299 unsigned long vaddr, addr, mapping_size = 0; 301 300 bool prev_exec, exec = false; ··· 304 301 int psize; 305 302 unsigned long max_mapping_size = memory_block_size; 306 303 307 - if (debug_pagealloc_enabled_or_kfence()) 304 + if (mapping_sz_limit < max_mapping_size) 305 + max_mapping_size = mapping_sz_limit; 306 + 307 + if (debug_pagealloc_enabled()) 308 308 max_mapping_size = PAGE_SIZE; 309 309 310 310 start = ALIGN(start, PAGE_SIZE); ··· 362 356 return 0; 363 357 } 364 358 359 + #ifdef CONFIG_KFENCE 360 + static bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL; 361 + 362 + static int __init parse_kfence_early_init(char *arg) 363 + { 364 + int val; 365 + 366 + if (get_option(&arg, &val)) 367 + kfence_early_init = !!val; 368 + return 0; 369 + } 370 + early_param("kfence.sample_interval", parse_kfence_early_init); 371 + 372 + static inline phys_addr_t alloc_kfence_pool(void) 373 + { 374 + phys_addr_t kfence_pool; 375 + 376 + /* 377 + * TODO: Support to enable KFENCE after bootup depends on the ability to 378 + * split page table mappings. As such support is not currently 379 + * implemented for radix pagetables, support enabling KFENCE 380 + * only at system startup for now. 381 + * 382 + * After support for splitting mappings is available on radix, 383 + * alloc_kfence_pool() & map_kfence_pool() can be dropped and 384 + * mapping for __kfence_pool memory can be 385 + * split during arch_kfence_init_pool(). 386 + */ 387 + if (!kfence_early_init) 388 + goto no_kfence; 389 + 390 + kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); 391 + if (!kfence_pool) 392 + goto no_kfence; 393 + 394 + memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); 395 + return kfence_pool; 396 + 397 + no_kfence: 398 + disable_kfence(); 399 + return 0; 400 + } 401 + 402 + static inline void map_kfence_pool(phys_addr_t kfence_pool) 403 + { 404 + if (!kfence_pool) 405 + return; 406 + 407 + if (create_physical_mapping(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, 408 + -1, PAGE_KERNEL, PAGE_SIZE)) 409 + goto err; 410 + 411 + memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); 412 + __kfence_pool = __va(kfence_pool); 413 + return; 414 + 415 + err: 416 + memblock_phys_free(kfence_pool, KFENCE_POOL_SIZE); 417 + disable_kfence(); 418 + } 419 + #else 420 + static inline phys_addr_t alloc_kfence_pool(void) { return 0; } 421 + static inline void map_kfence_pool(phys_addr_t kfence_pool) { } 422 + #endif 423 + 365 424 static void __init radix_init_pgtable(void) 366 425 { 426 + phys_addr_t kfence_pool; 367 427 unsigned long rts_field; 368 428 phys_addr_t start, end; 369 429 u64 i; 370 430 371 431 /* We don't support slb for radix */ 372 432 slb_set_size(0); 433 + 434 + kfence_pool = alloc_kfence_pool(); 373 435 374 436 /* 375 437 * Create the linear mapping ··· 455 381 } 456 382 457 383 WARN_ON(create_physical_mapping(start, end, 458 - -1, PAGE_KERNEL)); 384 + -1, PAGE_KERNEL, ~0UL)); 459 385 } 386 + 387 + map_kfence_pool(kfence_pool); 460 388 461 389 if (!cpu_has_feature(CPU_FTR_HVMODE) && 462 390 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) { ··· 951 875 } 952 876 953 877 return create_physical_mapping(__pa(start), __pa(end), 954 - nid, prot); 878 + nid, prot, ~0UL); 955 879 } 956 880 957 881 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)

arch/powerpc/mm/init-common.c

··· 31 31 32 32 bool disable_kuep = !IS_ENABLED(CONFIG_PPC_KUEP); 33 33 bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP); 34 + #ifdef CONFIG_KFENCE 35 + bool __ro_after_init kfence_disabled; 36 + #endif 34 37 35 38 static int __init parse_nosmep(char *p) 36 39 {

Configure Feed

Configure Feed