Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'slab-for-7.0-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab
Pull slab fixes from Vlastimil Babka:
- Fix for spurious page allocation warnings on sheaf refill (Harry Yoo)
- Fix for CONFIG_MEM_ALLOC_PROFILING_DEBUG warnings (Suren
Baghdasaryan)
- Fix for kernel-doc warning on ksize() (Sanjay Chitroda)
- Fix to avoid setting slab->stride later than on slab allocation.
Doesn't yet fix the reports from powerpc; debugging is making
progress (Harry Yoo)
* tag 'slab-for-7.0-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
mm/slab: initialize slab->stride early to avoid memory ordering issues
mm/slub: drop duplicate kernel-doc for ksize()
mm/slab: mark alloc tags empty for sheaves allocated with __GFP_NO_OBJ_EXT
mm/slab: pass __GFP_NOWARN to refill_sheaf() if fallback is available
+39
-30
+2
include/linux/gfp_types.h
+2
include/linux/gfp_types.h
···
139
139
* %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
140
140
*
141
141
* %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension.
142
+
* mark_obj_codetag_empty() should be called upon freeing for objects allocated
143
+
* with this flag to indicate that their NULL tags are expected and normal.
142
144
*/
143
145
#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
144
146
#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE)
-12
include/linux/slab.h
-12
include/linux/slab.h
···
517
517
DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
518
518
DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T))
519
519
520
-
/**
521
-
* ksize - Report actual allocation size of associated object
522
-
*
523
-
* @objp: Pointer returned from a prior kmalloc()-family allocation.
524
-
*
525
-
* This should not be used for writing beyond the originally requested
526
-
* allocation size. Either use krealloc() or round up the allocation size
527
-
* with kmalloc_size_roundup() prior to allocation. If this is used to
528
-
* access beyond the originally requested allocation size, UBSAN_BOUNDS
529
-
* and/or FORTIFY_SOURCE may trip, since they only know about the
530
-
* originally allocated size via the __alloc_size attribute.
531
-
*/
532
520
size_t ksize(const void *objp);
533
521
534
522
#ifdef CONFIG_PRINTK
+2
-2
mm/slab.h
+2
-2
mm/slab.h
···
290
290
291
291
/* Determine object index from a given position */
292
292
static inline unsigned int __obj_to_index(const struct kmem_cache *cache,
293
-
void *addr, void *obj)
293
+
void *addr, const void *obj)
294
294
{
295
295
return reciprocal_divide(kasan_reset_tag(obj) - addr,
296
296
cache->reciprocal_size);
297
297
}
298
298
299
299
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
300
-
const struct slab *slab, void *obj)
300
+
const struct slab *slab, const void *obj)
301
301
{
302
302
if (is_kfence_address(obj))
303
303
return 0;
+35
-16
mm/slub.c
+35
-16
mm/slub.c
···
2041
2041
2042
2042
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
2043
2043
2044
-
static inline void mark_objexts_empty(struct slabobj_ext *obj_exts)
2044
+
static inline void mark_obj_codetag_empty(const void *obj)
2045
2045
{
2046
-
struct slab *obj_exts_slab;
2046
+
struct slab *obj_slab;
2047
2047
unsigned long slab_exts;
2048
2048
2049
-
obj_exts_slab = virt_to_slab(obj_exts);
2050
-
slab_exts = slab_obj_exts(obj_exts_slab);
2049
+
obj_slab = virt_to_slab(obj);
2050
+
slab_exts = slab_obj_exts(obj_slab);
2051
2051
if (slab_exts) {
2052
2052
get_slab_obj_exts(slab_exts);
2053
-
unsigned int offs = obj_to_index(obj_exts_slab->slab_cache,
2054
-
obj_exts_slab, obj_exts);
2055
-
struct slabobj_ext *ext = slab_obj_ext(obj_exts_slab,
2053
+
unsigned int offs = obj_to_index(obj_slab->slab_cache,
2054
+
obj_slab, obj);
2055
+
struct slabobj_ext *ext = slab_obj_ext(obj_slab,
2056
2056
slab_exts, offs);
2057
2057
2058
2058
if (unlikely(is_codetag_empty(&ext->ref))) {
···
2090
2090
2091
2091
#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
2092
2092
2093
-
static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {}
2093
+
static inline void mark_obj_codetag_empty(const void *obj) {}
2094
2094
static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; }
2095
2095
static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
2096
2096
struct slabobj_ext *vec, unsigned int objects) {}
···
2196
2196
retry:
2197
2197
old_exts = READ_ONCE(slab->obj_exts);
2198
2198
handle_failed_objexts_alloc(old_exts, vec, objects);
2199
-
slab_set_stride(slab, sizeof(struct slabobj_ext));
2200
2199
2201
2200
if (new_slab) {
2202
2201
/*
···
2210
2211
* assign slabobj_exts in parallel. In this case the existing
2211
2212
* objcg vector should be reused.
2212
2213
*/
2213
-
mark_objexts_empty(vec);
2214
+
mark_obj_codetag_empty(vec);
2214
2215
if (unlikely(!allow_spin))
2215
2216
kfree_nolock(vec);
2216
2217
else
···
2253
2254
* NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
2254
2255
* the extension for obj_exts is expected to be NULL.
2255
2256
*/
2256
-
mark_objexts_empty(obj_exts);
2257
+
mark_obj_codetag_empty(obj_exts);
2257
2258
if (allow_spin)
2258
2259
kfree(obj_exts);
2259
2260
else
···
2271
2272
void *addr;
2272
2273
unsigned long obj_exts;
2273
2274
2275
+
/* Initialize stride early to avoid memory ordering issues */
2276
+
slab_set_stride(slab, sizeof(struct slabobj_ext));
2277
+
2274
2278
if (!need_slab_obj_exts(s))
2275
2279
return;
2276
2280
···
2290
2288
obj_exts |= MEMCG_DATA_OBJEXTS;
2291
2289
#endif
2292
2290
slab->obj_exts = obj_exts;
2293
-
slab_set_stride(slab, sizeof(struct slabobj_ext));
2294
2291
} else if (s->flags & SLAB_OBJ_EXT_IN_OBJ) {
2295
2292
unsigned int offset = obj_exts_offset_in_object(s);
2296
2293
···
2312
2311
}
2313
2312
2314
2313
#else /* CONFIG_SLAB_OBJ_EXT */
2314
+
2315
+
static inline void mark_obj_codetag_empty(const void *obj)
2316
+
{
2317
+
}
2315
2318
2316
2319
static inline void init_slab_obj_exts(struct slab *slab)
2317
2320
{
···
2788
2783
2789
2784
static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
2790
2785
{
2786
+
/*
2787
+
* If the sheaf was created with __GFP_NO_OBJ_EXT flag then its
2788
+
* corresponding extension is NULL and alloc_tag_sub() will throw a
2789
+
* warning, therefore replace NULL with CODETAG_EMPTY to indicate
2790
+
* that the extension for this sheaf is expected to be NULL.
2791
+
*/
2792
+
if (s->flags & SLAB_KMALLOC)
2793
+
mark_obj_codetag_empty(sheaf);
2794
+
2791
2795
kfree(sheaf);
2792
2796
2793
2797
stat(s, SHEAF_FREE);
···
2836
2822
if (!sheaf)
2837
2823
return NULL;
2838
2824
2839
-
if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC)) {
2825
+
if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
2840
2826
free_empty_sheaf(s, sheaf);
2841
2827
return NULL;
2842
2828
}
···
4589
4575
return NULL;
4590
4576
4591
4577
if (empty) {
4592
-
if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC)) {
4578
+
if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
4593
4579
full = empty;
4594
4580
} else {
4595
4581
/*
···
4904
4890
static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
4905
4891
struct slab_sheaf *sheaf, gfp_t gfp)
4906
4892
{
4907
-
int ret = 0;
4893
+
gfp_t gfp_nomemalloc;
4894
+
int ret;
4908
4895
4909
-
ret = refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC);
4896
+
gfp_nomemalloc = gfp | __GFP_NOMEMALLOC;
4897
+
if (gfp_pfmemalloc_allowed(gfp))
4898
+
gfp_nomemalloc |= __GFP_NOWARN;
4899
+
4900
+
ret = refill_sheaf(s, sheaf, gfp_nomemalloc);
4910
4901
4911
4902
if (likely(!ret || !gfp_pfmemalloc_allowed(gfp)))
4912
4903
return ret;