tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

LoongArch: Add 128-bit atomic cmpxchg support

Implement 128-bit atomic compare-and-exchange using LoongArch's
LL.D/SC.Q instructions.

At the same time, this fix the BPF scheduler test failures (scx_central
and scx_qmap) caused by kmalloc_nolock_noprof() returning NULL, due to
missing 128-bit atomics. The NULL returns lead to -ENOMEM errors during
scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler
(located in tools/sched_ext/).

Building with `make` and running ./tools/sched_ext/build/bin/scx_qmap.

Link: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/commit/?id=5fb750e8a9ae
Acked-by: Hengqi Chen <hengqi.chen@gmail.com>
Tested-by: Hengqi Chen <hengqi.chen@gmail.com>
Co-developed-by:: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

authored by

George Guo and committed by
Huacai Chen f0e4b1b6 48543c42

+56
+2
arch/loongarch/Kconfig
··· 114 114 select GENERIC_TIME_VSYSCALL 115 115 select GPIOLIB 116 116 select HAS_IOPORT 117 + select HAVE_ALIGNED_STRUCT_PAGE 117 118 select HAVE_ARCH_AUDITSYSCALL 118 119 select HAVE_ARCH_BITREVERSE 119 120 select HAVE_ARCH_JUMP_LABEL ··· 131 130 select HAVE_ARCH_TRANSPARENT_HUGEPAGE 132 131 select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD 133 132 select HAVE_ASM_MODVERSIONS 133 + select HAVE_CMPXCHG_DOUBLE 134 134 select HAVE_CMPXCHG_LOCAL 135 135 select HAVE_CONTEXT_TRACKING_USER 136 136 select HAVE_C_RECORDMCOUNT
+54
arch/loongarch/include/asm/cmpxchg.h
··· 8 8 #include <linux/bits.h> 9 9 #include <linux/build_bug.h> 10 10 #include <asm/barrier.h> 11 + #include <asm/cpu-features.h> 11 12 12 13 #define __xchg_amo_asm(amswap_db, m, val) \ 13 14 ({ \ ··· 237 236 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 238 237 arch_cmpxchg((ptr), (o), (n)); \ 239 238 }) 239 + 240 + union __u128_halves { 241 + u128 full; 242 + struct { 243 + u64 low; 244 + u64 high; 245 + }; 246 + }; 247 + 248 + #define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ) 249 + 250 + #define __arch_cmpxchg128(ptr, old, new, llsc_mb) \ 251 + ({ \ 252 + union __u128_halves __old, __new, __ret; \ 253 + volatile u64 *__ptr = (volatile u64 *)(ptr); \ 254 + \ 255 + __old.full = (old); \ 256 + __new.full = (new); \ 257 + \ 258 + __asm__ __volatile__( \ 259 + "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \ 260 + llsc_mb \ 261 + " ld.d %1, %4 # 128-bit cmpxchg high \n" \ 262 + " move $t0, %0 \n" \ 263 + " move $t1, %1 \n" \ 264 + " bne %0, %z5, 2f \n" \ 265 + " bne %1, %z6, 2f \n" \ 266 + " move $t0, %z7 \n" \ 267 + " move $t1, %z8 \n" \ 268 + "2: sc.q $t0, $t1, %2 \n" \ 269 + " beqz $t0, 1b \n" \ 270 + llsc_mb \ 271 + : "=&r" (__ret.low), "=&r" (__ret.high) \ 272 + : "r" (__ptr), \ 273 + "ZC" (__ptr[0]), "m" (__ptr[1]), \ 274 + "Jr" (__old.low), "Jr" (__old.high), \ 275 + "Jr" (__new.low), "Jr" (__new.high) \ 276 + : "t0", "t1", "memory"); \ 277 + \ 278 + __ret.full; \ 279 + }) 280 + 281 + #define arch_cmpxchg128(ptr, o, n) \ 282 + ({ \ 283 + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ 284 + __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \ 285 + }) 286 + 287 + #define arch_cmpxchg128_local(ptr, o, n) \ 288 + ({ \ 289 + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ 290 + __arch_cmpxchg128(ptr, o, n, ""); \ 291 + }) 240 292 #else 241 293 #include <asm-generic/cmpxchg-local.h> 242 294 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))