Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
bitops: add xor_unlock_is_negative_byte()
Replace clear_bit_and_unlock_is_negative_byte() with
xor_unlock_is_negative_byte(). We have a few places that like to lock a
folio, set a flag and unlock it again. Allow for the possibility of
combining the latter two operations for efficiency. We are guaranteed
that the caller holds the lock, so it is safe to unlock it with the xor.
The caller must guarantee that nobody else will set the flag without
holding the lock; it is not safe to do this with the PG_dirty flag, for
example.
Link: https://lkml.kernel.org/r/20231004165317.1061855-8-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Andreas Dilger <adilger.kernel@dilger.ca>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Richard Henderson <richard.henderson@linaro.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: "Theodore Ts'o" <tytso@mit.edu>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Matthew Wilcox (Oracle)
and committed by
Andrew Morton
247dbcdb
7a4847e5
+47
-57
+5
-12
arch/powerpc/include/asm/bitops.h
+5
-12
arch/powerpc/include/asm/bitops.h
···
234
234
}
235
235
236
236
#ifdef CONFIG_PPC64
237
-
static inline unsigned long
238
-
clear_bit_unlock_return_word(int nr, volatile unsigned long *addr)
237
+
static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
238
+
volatile unsigned long *p)
239
239
{
240
240
unsigned long old, t;
241
-
unsigned long *p = (unsigned long *)addr + BIT_WORD(nr);
242
-
unsigned long mask = BIT_MASK(nr);
243
241
244
242
__asm__ __volatile__ (
245
243
PPC_RELEASE_BARRIER
246
244
"1:" PPC_LLARX "%0,0,%3,0\n"
247
-
"andc %1,%0,%2\n"
245
+
"xor %1,%0,%2\n"
248
246
PPC_STLCX "%1,0,%3\n"
249
247
"bne- 1b\n"
250
248
: "=&r" (old), "=&r" (t)
251
249
: "r" (mask), "r" (p)
252
250
: "cc", "memory");
253
251
254
-
return old;
252
+
return (old & BIT_MASK(7)) != 0;
255
253
}
256
254
257
-
/*
258
-
* This is a special function for mm/filemap.c
259
-
* Bit 7 corresponds to PG_waiters.
260
-
*/
261
-
#define arch_clear_bit_unlock_is_negative_byte(nr, addr) \
262
-
(clear_bit_unlock_return_word(nr, addr) & BIT_MASK(7))
255
+
#define arch_xor_unlock_is_negative_byte arch_xor_unlock_is_negative_byte
263
256
264
257
#endif /* CONFIG_PPC64 */
265
258
+5
-6
arch/x86/include/asm/bitops.h
+5
-6
arch/x86/include/asm/bitops.h
···
94
94
asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
95
95
}
96
96
97
-
static __always_inline bool
98
-
arch_clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
97
+
static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
98
+
volatile unsigned long *addr)
99
99
{
100
100
bool negative;
101
-
asm volatile(LOCK_PREFIX "andb %2,%1"
101
+
asm volatile(LOCK_PREFIX "xorb %2,%1"
102
102
CC_SET(s)
103
103
: CC_OUT(s) (negative), WBYTE_ADDR(addr)
104
-
: "ir" ((char) ~(1 << nr)) : "memory");
104
+
: "iq" ((char)mask) : "memory");
105
105
return negative;
106
106
}
107
-
#define arch_clear_bit_unlock_is_negative_byte \
108
-
arch_clear_bit_unlock_is_negative_byte
107
+
#define arch_xor_unlock_is_negative_byte arch_xor_unlock_is_negative_byte
109
108
110
109
static __always_inline void
111
110
arch___clear_bit_unlock(long nr, volatile unsigned long *addr)
+15
-12
include/asm-generic/bitops/instrumented-lock.h
+15
-12
include/asm-generic/bitops/instrumented-lock.h
···
58
58
return arch_test_and_set_bit_lock(nr, addr);
59
59
}
60
60
61
-
#if defined(arch_clear_bit_unlock_is_negative_byte)
61
+
#if defined(arch_xor_unlock_is_negative_byte)
62
62
/**
63
-
* clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
64
-
* byte is negative, for unlock.
65
-
* @nr: the bit to clear
66
-
* @addr: the address to start counting from
63
+
* xor_unlock_is_negative_byte - XOR a single byte in memory and test if
64
+
* it is negative, for unlock.
65
+
* @mask: Change the bits which are set in this mask.
66
+
* @addr: The address of the word containing the byte to change.
67
67
*
68
+
* Changes some of bits 0-6 in the word pointed to by @addr.
68
69
* This operation is atomic and provides release barrier semantics.
70
+
* Used to optimise some folio operations which are commonly paired
71
+
* with an unlock or end of writeback. Bit 7 is used as PG_waiters to
72
+
* indicate whether anybody is waiting for the unlock.
69
73
*
70
-
* This is a bit of a one-trick-pony for the filemap code, which clears
71
-
* PG_locked and tests PG_waiters,
74
+
* Return: Whether the top bit of the byte is set.
72
75
*/
73
-
static inline bool
74
-
clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
76
+
static inline bool xor_unlock_is_negative_byte(unsigned long mask,
77
+
volatile unsigned long *addr)
75
78
{
76
79
kcsan_release();
77
-
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
78
-
return arch_clear_bit_unlock_is_negative_byte(nr, addr);
80
+
instrument_atomic_write(addr, sizeof(long));
81
+
return arch_xor_unlock_is_negative_byte(mask, addr);
79
82
}
80
83
/* Let everybody know we have it. */
81
-
#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte
84
+
#define xor_unlock_is_negative_byte xor_unlock_is_negative_byte
82
85
#endif
83
86
84
87
#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */
+5
-16
include/asm-generic/bitops/lock.h
+5
-16
include/asm-generic/bitops/lock.h
···
66
66
raw_atomic_long_set_release((atomic_long_t *)p, old);
67
67
}
68
68
69
-
/**
70
-
* arch_clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
71
-
* byte is negative, for unlock.
72
-
* @nr: the bit to clear
73
-
* @addr: the address to start counting from
74
-
*
75
-
* This is a bit of a one-trick-pony for the filemap code, which clears
76
-
* PG_locked and tests PG_waiters,
77
-
*/
78
-
#ifndef arch_clear_bit_unlock_is_negative_byte
79
-
static inline bool arch_clear_bit_unlock_is_negative_byte(unsigned int nr,
80
-
volatile unsigned long *p)
69
+
#ifndef arch_xor_unlock_is_negative_byte
70
+
static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
71
+
volatile unsigned long *p)
81
72
{
82
73
long old;
83
-
unsigned long mask = BIT_MASK(nr);
84
74
85
-
p += BIT_WORD(nr);
86
-
old = raw_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
75
+
old = raw_atomic_long_fetch_xor_release(mask, (atomic_long_t *)p);
87
76
return !!(old & BIT(7));
88
77
}
89
-
#define arch_clear_bit_unlock_is_negative_byte arch_clear_bit_unlock_is_negative_byte
78
+
#define arch_xor_unlock_is_negative_byte arch_xor_unlock_is_negative_byte
90
79
#endif
91
80
92
81
#include <asm-generic/bitops/instrumented-lock.h>
+4
-4
kernel/kcsan/kcsan_test.c
+4
-4
kernel/kcsan/kcsan_test.c
···
700
700
KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false);
701
701
KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true);
702
702
703
-
#ifdef clear_bit_unlock_is_negative_byte
704
-
KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
705
-
KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
706
-
KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
703
+
#ifdef xor_unlock_is_negative_byte
704
+
KCSAN_EXPECT_READ_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
705
+
KCSAN_EXPECT_WRITE_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
706
+
KCSAN_EXPECT_RW_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
707
707
#endif
708
708
kcsan_nestable_atomic_end();
709
709
}
+4
-4
kernel/kcsan/selftest.c
+4
-4
kernel/kcsan/selftest.c
···
228
228
spin_lock(&test_spinlock);
229
229
KCSAN_CHECK_RW_BARRIER(spin_unlock(&test_spinlock));
230
230
231
-
#ifdef clear_bit_unlock_is_negative_byte
232
-
KCSAN_CHECK_RW_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var));
233
-
KCSAN_CHECK_READ_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var));
234
-
KCSAN_CHECK_WRITE_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var));
231
+
#ifdef xor_unlock_is_negative_byte
232
+
KCSAN_CHECK_RW_BARRIER(xor_unlock_is_negative_byte(1, &test_var));
233
+
KCSAN_CHECK_READ_BARRIER(xor_unlock_is_negative_byte(1, &test_var));
234
+
KCSAN_CHECK_WRITE_BARRIER(xor_unlock_is_negative_byte(1, &test_var));
235
235
#endif
236
236
kcsan_nestable_atomic_end();
237
237
+5
mm/filemap.c
+5
mm/filemap.c
···
1482
1482
}
1483
1483
EXPORT_SYMBOL_GPL(folio_add_wait_queue);
1484
1484
1485
+
#ifdef xor_unlock_is_negative_byte
1486
+
#define clear_bit_unlock_is_negative_byte(nr, p) \
1487
+
xor_unlock_is_negative_byte(1 << nr, p)
1488
+
#endif
1489
+
1485
1490
#ifndef clear_bit_unlock_is_negative_byte
1486
1491
1487
1492
/*
+4
-3
mm/kasan/kasan_test.c
+4
-3
mm/kasan/kasan_test.c
···
1099
1099
KUNIT_EXPECT_KASAN_FAIL(test, __test_and_change_bit(nr, addr));
1100
1100
KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = test_bit(nr, addr));
1101
1101
1102
-
#if defined(clear_bit_unlock_is_negative_byte)
1103
-
KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result =
1104
-
clear_bit_unlock_is_negative_byte(nr, addr));
1102
+
#if defined(xor_unlock_is_negative_byte)
1103
+
if (nr < 7)
1104
+
KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result =
1105
+
xor_unlock_is_negative_byte(1 << nr, addr));
1105
1106
#endif
1106
1107
}
1107
1108