Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
4 *
5 * (C) SGI 2006, Christoph Lameter
6 * Cleaned up and restructured to ease the addition of alternative
7 * implementations of SLAB allocators.
8 * (C) Linux Foundation 2008-2013
9 * Unified interface for all slab allocators
10 */
11
12#ifndef _LINUX_SLAB_H
13#define _LINUX_SLAB_H
14
15#include <linux/bug.h>
16#include <linux/cache.h>
17#include <linux/gfp.h>
18#include <linux/overflow.h>
19#include <linux/types.h>
20#include <linux/rcupdate.h>
21#include <linux/workqueue.h>
22#include <linux/percpu-refcount.h>
23#include <linux/cleanup.h>
24#include <linux/hash.h>
25
26enum _slab_flag_bits {
27 _SLAB_CONSISTENCY_CHECKS,
28 _SLAB_RED_ZONE,
29 _SLAB_POISON,
30 _SLAB_KMALLOC,
31 _SLAB_HWCACHE_ALIGN,
32 _SLAB_CACHE_DMA,
33 _SLAB_CACHE_DMA32,
34 _SLAB_STORE_USER,
35 _SLAB_PANIC,
36 _SLAB_TYPESAFE_BY_RCU,
37 _SLAB_TRACE,
38#ifdef CONFIG_DEBUG_OBJECTS
39 _SLAB_DEBUG_OBJECTS,
40#endif
41 _SLAB_NOLEAKTRACE,
42 _SLAB_NO_MERGE,
43#ifdef CONFIG_FAILSLAB
44 _SLAB_FAILSLAB,
45#endif
46#ifdef CONFIG_MEMCG
47 _SLAB_ACCOUNT,
48#endif
49#ifdef CONFIG_KASAN_GENERIC
50 _SLAB_KASAN,
51#endif
52 _SLAB_NO_USER_FLAGS,
53#ifdef CONFIG_KFENCE
54 _SLAB_SKIP_KFENCE,
55#endif
56#ifndef CONFIG_SLUB_TINY
57 _SLAB_RECLAIM_ACCOUNT,
58#endif
59 _SLAB_OBJECT_POISON,
60 _SLAB_CMPXCHG_DOUBLE,
61#ifdef CONFIG_SLAB_OBJ_EXT
62 _SLAB_NO_OBJ_EXT,
63#endif
64 _SLAB_FLAGS_LAST_BIT
65};
66
67#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
68#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
69
70/*
71 * Flags to pass to kmem_cache_create().
72 * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op
73 */
74/* DEBUG: Perform (expensive) checks on alloc/free */
75#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
76/* DEBUG: Red zone objs in a cache */
77#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
78/* DEBUG: Poison objects */
79#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
80/* Indicate a kmalloc slab */
81#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
82/**
83 * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries.
84 *
85 * Sufficiently large objects are aligned on cache line boundary. For object
86 * size smaller than a half of cache line size, the alignment is on the half of
87 * cache line size. In general, if object size is smaller than 1/2^n of cache
88 * line size, the alignment is adjusted to 1/2^n.
89 *
90 * If explicit alignment is also requested by the respective
91 * &struct kmem_cache_args field, the greater of both is alignments is applied.
92 */
93#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
94/* Use GFP_DMA memory */
95#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
96/* Use GFP_DMA32 memory */
97#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
98/* DEBUG: Store the last owner for bug hunting */
99#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
100/* Panic if kmem_cache_create() fails */
101#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
102/**
103 * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
104 *
105 * This delays freeing the SLAB page by a grace period, it does _NOT_
106 * delay object freeing. This means that if you do kmem_cache_free()
107 * that memory location is free to be reused at any time. Thus it may
108 * be possible to see another object there in the same RCU grace period.
109 *
110 * This feature only ensures the memory location backing the object
111 * stays valid, the trick to using this is relying on an independent
112 * object validation pass. Something like:
113 *
114 * ::
115 *
116 * begin:
117 * rcu_read_lock();
118 * obj = lockless_lookup(key);
119 * if (obj) {
120 * if (!try_get_ref(obj)) // might fail for free objects
121 * rcu_read_unlock();
122 * goto begin;
123 *
124 * if (obj->key != key) { // not the object we expected
125 * put_ref(obj);
126 * rcu_read_unlock();
127 * goto begin;
128 * }
129 * }
130 * rcu_read_unlock();
131 *
132 * This is useful if we need to approach a kernel structure obliquely,
133 * from its address obtained without the usual locking. We can lock
134 * the structure to stabilize it and check it's still at the given address,
135 * only if we can be sure that the memory has not been meanwhile reused
136 * for some other kind of object (which our subsystem's lock might corrupt).
137 *
138 * rcu_read_lock before reading the address, then rcu_read_unlock after
139 * taking the spinlock within the structure expected at that address.
140 *
141 * Note that object identity check has to be done *after* acquiring a
142 * reference, therefore user has to ensure proper ordering for loads.
143 * Similarly, when initializing objects allocated with SLAB_TYPESAFE_BY_RCU,
144 * the newly allocated object has to be fully initialized *before* its
145 * refcount gets initialized and proper ordering for stores is required.
146 * refcount_{add|inc}_not_zero_acquire() and refcount_set_release() are
147 * designed with the proper fences required for reference counting objects
148 * allocated with SLAB_TYPESAFE_BY_RCU.
149 *
150 * Note that it is not possible to acquire a lock within a structure
151 * allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
152 * as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
153 * are not zeroed before being given to the slab, which means that any
154 * locks must be initialized after each and every kmem_struct_alloc().
155 * Alternatively, make the ctor passed to kmem_cache_create() initialize
156 * the locks at page-allocation time, as is done in __i915_request_ctor(),
157 * sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
158 * to safely acquire those ctor-initialized locks under rcu_read_lock()
159 * protection.
160 *
161 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
162 */
163#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
164/* Trace allocations and frees */
165#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
166
167/* Flag to prevent checks on free */
168#ifdef CONFIG_DEBUG_OBJECTS
169# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
170#else
171# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
172#endif
173
174/* Avoid kmemleak tracing */
175#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
176
177/*
178 * Prevent merging with compatible kmem caches. This flag should be used
179 * cautiously. Valid use cases:
180 *
181 * - caches created for self-tests (e.g. kunit)
182 * - general caches created and used by a subsystem, only when a
183 * (subsystem-specific) debug option is enabled
184 * - performance critical caches, should be very rare and consulted with slab
185 * maintainers, and not used together with CONFIG_SLUB_TINY
186 */
187#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
188
189/* Fault injection mark */
190#ifdef CONFIG_FAILSLAB
191# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
192#else
193# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
194#endif
195/**
196 * define SLAB_ACCOUNT - Account allocations to memcg.
197 *
198 * All object allocations from this cache will be memcg accounted, regardless of
199 * __GFP_ACCOUNT being or not being passed to individual allocations.
200 */
201#ifdef CONFIG_MEMCG
202# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
203#else
204# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
205#endif
206
207#ifdef CONFIG_KASAN_GENERIC
208#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
209#else
210#define SLAB_KASAN __SLAB_FLAG_UNUSED
211#endif
212
213/*
214 * Ignore user specified debugging flags.
215 * Intended for caches created for self-tests so they have only flags
216 * specified in the code and other flags are ignored.
217 */
218#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
219
220#ifdef CONFIG_KFENCE
221#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
222#else
223#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
224#endif
225
226/* The following flags affect the page allocator grouping pages by mobility */
227/**
228 * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable.
229 *
230 * Use this flag for caches that have an associated shrinker. As a result, slab
231 * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by
232 * mobility, and are accounted in SReclaimable counter in /proc/meminfo
233 */
234#ifndef CONFIG_SLUB_TINY
235#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
236#else
237#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
238#endif
239#define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
240
241/* Slab created using create_boot_cache */
242#ifdef CONFIG_SLAB_OBJ_EXT
243#define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
244#else
245#define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED
246#endif
247
248/*
249 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
250 *
251 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
252 *
253 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
254 * Both make kfree a no-op.
255 */
256#define ZERO_SIZE_PTR ((void *)16)
257
258#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
259 (unsigned long)ZERO_SIZE_PTR)
260
261#include <linux/kasan.h>
262
263struct list_lru;
264struct mem_cgroup;
265/*
266 * struct kmem_cache related prototypes
267 */
268bool slab_is_available(void);
269
270/**
271 * struct kmem_cache_args - Less common arguments for kmem_cache_create()
272 *
273 * Any uninitialized fields of the structure are interpreted as unused. The
274 * exception is @freeptr_offset where %0 is a valid value, so
275 * @use_freeptr_offset must be also set to %true in order to interpret the field
276 * as used. For @useroffset %0 is also valid, but only with non-%0
277 * @usersize.
278 *
279 * When %NULL args is passed to kmem_cache_create(), it is equivalent to all
280 * fields unused.
281 */
282struct kmem_cache_args {
283 /**
284 * @align: The required alignment for the objects.
285 *
286 * %0 means no specific alignment is requested.
287 */
288 unsigned int align;
289 /**
290 * @useroffset: Usercopy region offset.
291 *
292 * %0 is a valid offset, when @usersize is non-%0
293 */
294 unsigned int useroffset;
295 /**
296 * @usersize: Usercopy region size.
297 *
298 * %0 means no usercopy region is specified.
299 */
300 unsigned int usersize;
301 /**
302 * @freeptr_offset: Custom offset for the free pointer
303 * in &SLAB_TYPESAFE_BY_RCU caches
304 *
305 * By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer
306 * outside of the object. This might cause the object to grow in size.
307 * Cache creators that have a reason to avoid this can specify a custom
308 * free pointer offset in their struct where the free pointer will be
309 * placed.
310 *
311 * Note that placing the free pointer inside the object requires the
312 * caller to ensure that no fields are invalidated that are required to
313 * guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for
314 * details).
315 *
316 * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
317 * is specified, %use_freeptr_offset must be set %true.
318 *
319 * Note that @ctor currently isn't supported with custom free pointers
320 * as a @ctor requires an external free pointer.
321 */
322 unsigned int freeptr_offset;
323 /**
324 * @use_freeptr_offset: Whether a @freeptr_offset is used.
325 */
326 bool use_freeptr_offset;
327 /**
328 * @ctor: A constructor for the objects.
329 *
330 * The constructor is invoked for each object in a newly allocated slab
331 * page. It is the cache user's responsibility to free object in the
332 * same state as after calling the constructor, or deal appropriately
333 * with any differences between a freshly constructed and a reallocated
334 * object.
335 *
336 * %NULL means no constructor.
337 */
338 void (*ctor)(void *);
339 /**
340 * @sheaf_capacity: Enable sheaves of given capacity for the cache.
341 *
342 * With a non-zero value, allocations from the cache go through caching
343 * arrays called sheaves. Each cpu has a main sheaf that's always
344 * present, and a spare sheaf that may be not present. When both become
345 * empty, there's an attempt to replace an empty sheaf with a full sheaf
346 * from the per-node barn.
347 *
348 * When no full sheaf is available, and gfp flags allow blocking, a
349 * sheaf is allocated and filled from slab(s) using bulk allocation.
350 * Otherwise the allocation falls back to the normal operation
351 * allocating a single object from a slab.
352 *
353 * Analogically when freeing and both percpu sheaves are full, the barn
354 * may replace it with an empty sheaf, unless it's over capacity. In
355 * that case a sheaf is bulk freed to slab pages.
356 *
357 * The sheaves do not enforce NUMA placement of objects, so allocations
358 * via kmem_cache_alloc_node() with a node specified other than
359 * NUMA_NO_NODE will bypass them.
360 *
361 * Bulk allocation and free operations also try to use the cpu sheaves
362 * and barn, but fallback to using slab pages directly.
363 *
364 * When slub_debug is enabled for the cache, the sheaf_capacity argument
365 * is ignored.
366 *
367 * %0 means no sheaves will be created.
368 */
369 unsigned int sheaf_capacity;
370};
371
372struct kmem_cache *__kmem_cache_create_args(const char *name,
373 unsigned int object_size,
374 struct kmem_cache_args *args,
375 slab_flags_t flags);
376static inline struct kmem_cache *
377__kmem_cache_create(const char *name, unsigned int size, unsigned int align,
378 slab_flags_t flags, void (*ctor)(void *))
379{
380 struct kmem_cache_args kmem_args = {
381 .align = align,
382 .ctor = ctor,
383 };
384
385 return __kmem_cache_create_args(name, size, &kmem_args, flags);
386}
387
388/**
389 * kmem_cache_create_usercopy - Create a kmem cache with a region suitable
390 * for copying to userspace.
391 * @name: A string which is used in /proc/slabinfo to identify this cache.
392 * @size: The size of objects to be created in this cache.
393 * @align: The required alignment for the objects.
394 * @flags: SLAB flags
395 * @useroffset: Usercopy region offset
396 * @usersize: Usercopy region size
397 * @ctor: A constructor for the objects, or %NULL.
398 *
399 * This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY()
400 * if whitelisting a single field is sufficient, or kmem_cache_create() with
401 * the necessary parameters passed via the args parameter (see
402 * &struct kmem_cache_args)
403 *
404 * Return: a pointer to the cache on success, NULL on failure.
405 */
406static inline struct kmem_cache *
407kmem_cache_create_usercopy(const char *name, unsigned int size,
408 unsigned int align, slab_flags_t flags,
409 unsigned int useroffset, unsigned int usersize,
410 void (*ctor)(void *))
411{
412 struct kmem_cache_args kmem_args = {
413 .align = align,
414 .ctor = ctor,
415 .useroffset = useroffset,
416 .usersize = usersize,
417 };
418
419 return __kmem_cache_create_args(name, size, &kmem_args, flags);
420}
421
422/* If NULL is passed for @args, use this variant with default arguments. */
423static inline struct kmem_cache *
424__kmem_cache_default_args(const char *name, unsigned int size,
425 struct kmem_cache_args *args,
426 slab_flags_t flags)
427{
428 struct kmem_cache_args kmem_default_args = {};
429
430 /* Make sure we don't get passed garbage. */
431 if (WARN_ON_ONCE(args))
432 return ERR_PTR(-EINVAL);
433
434 return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
435}
436
437/**
438 * kmem_cache_create - Create a kmem cache.
439 * @__name: A string which is used in /proc/slabinfo to identify this cache.
440 * @__object_size: The size of objects to be created in this cache.
441 * @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL
442 * means defaults will be used for all the arguments.
443 *
444 * This is currently implemented as a macro using ``_Generic()`` to call
445 * either the new variant of the function, or a legacy one.
446 *
447 * The new variant has 4 parameters:
448 * ``kmem_cache_create(name, object_size, args, flags)``
449 *
450 * See __kmem_cache_create_args() which implements this.
451 *
452 * The legacy variant has 5 parameters:
453 * ``kmem_cache_create(name, object_size, align, flags, ctor)``
454 *
455 * The align and ctor parameters map to the respective fields of
456 * &struct kmem_cache_args
457 *
458 * Context: Cannot be called within a interrupt, but can be interrupted.
459 *
460 * Return: a pointer to the cache on success, NULL on failure.
461 */
462#define kmem_cache_create(__name, __object_size, __args, ...) \
463 _Generic((__args), \
464 struct kmem_cache_args *: __kmem_cache_create_args, \
465 void *: __kmem_cache_default_args, \
466 default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
467
468void kmem_cache_destroy(struct kmem_cache *s);
469int kmem_cache_shrink(struct kmem_cache *s);
470
471/*
472 * Please use this macro to create slab caches. Simply specify the
473 * name of the structure and maybe some flags that are listed above.
474 *
475 * The alignment of the struct determines object alignment. If you
476 * f.e. add ____cacheline_aligned_in_smp to the struct declaration
477 * then the objects will be properly aligned in SMP configurations.
478 */
479#define KMEM_CACHE(__struct, __flags) \
480 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
481 &(struct kmem_cache_args) { \
482 .align = __alignof__(struct __struct), \
483 }, (__flags))
484
485/*
486 * To whitelist a single field for copying to/from usercopy, use this
487 * macro instead for KMEM_CACHE() above.
488 */
489#define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
490 __kmem_cache_create_args(#__struct, sizeof(struct __struct), \
491 &(struct kmem_cache_args) { \
492 .align = __alignof__(struct __struct), \
493 .useroffset = offsetof(struct __struct, __field), \
494 .usersize = sizeof_field(struct __struct, __field), \
495 }, (__flags))
496
497/*
498 * Common kmalloc functions provided by all allocators
499 */
500void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size,
501 unsigned long align,
502 gfp_t flags, int nid) __realloc_size(2);
503#define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE)
504#define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__))
505#define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n)
506#define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE)
507
508void kfree(const void *objp);
509void kfree_nolock(const void *objp);
510void kfree_sensitive(const void *objp);
511size_t __ksize(const void *objp);
512
513DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
514DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T))
515
516/**
517 * ksize - Report actual allocation size of associated object
518 *
519 * @objp: Pointer returned from a prior kmalloc()-family allocation.
520 *
521 * This should not be used for writing beyond the originally requested
522 * allocation size. Either use krealloc() or round up the allocation size
523 * with kmalloc_size_roundup() prior to allocation. If this is used to
524 * access beyond the originally requested allocation size, UBSAN_BOUNDS
525 * and/or FORTIFY_SOURCE may trip, since they only know about the
526 * originally allocated size via the __alloc_size attribute.
527 */
528size_t ksize(const void *objp);
529
530#ifdef CONFIG_PRINTK
531bool kmem_dump_obj(void *object);
532#else
533static inline bool kmem_dump_obj(void *object) { return false; }
534#endif
535
536/*
537 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
538 * alignment larger than the alignment of a 64-bit integer.
539 * Setting ARCH_DMA_MINALIGN in arch headers allows that.
540 */
541#ifdef ARCH_HAS_DMA_MINALIGN
542#if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
543#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
544#endif
545#endif
546
547#ifndef ARCH_KMALLOC_MINALIGN
548#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
549#elif ARCH_KMALLOC_MINALIGN > 8
550#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
551#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
552#endif
553
554/*
555 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
556 * Intended for arches that get misalignment faults even for 64 bit integer
557 * aligned buffers.
558 */
559#ifndef ARCH_SLAB_MINALIGN
560#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
561#endif
562
563/*
564 * Arches can define this function if they want to decide the minimum slab
565 * alignment at runtime. The value returned by the function must be a power
566 * of two and >= ARCH_SLAB_MINALIGN.
567 */
568#ifndef arch_slab_minalign
569static inline unsigned int arch_slab_minalign(void)
570{
571 return ARCH_SLAB_MINALIGN;
572}
573#endif
574
575/*
576 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
577 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
578 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
579 */
580#define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
581#define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
582#define __assume_page_alignment __assume_aligned(PAGE_SIZE)
583
584/*
585 * Kmalloc array related definitions
586 */
587
588/*
589 * SLUB directly allocates requests fitting in to an order-1 page
590 * (PAGE_SIZE*2). Larger requests are passed to the page allocator.
591 */
592#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
593#define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT)
594#ifndef KMALLOC_SHIFT_LOW
595#define KMALLOC_SHIFT_LOW 3
596#endif
597
598/* Maximum allocatable size */
599#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
600/* Maximum size for which we actually use a slab cache */
601#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
602/* Maximum order allocatable via the slab allocator */
603#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
604
605/*
606 * Kmalloc subsystem.
607 */
608#ifndef KMALLOC_MIN_SIZE
609#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
610#endif
611
612/*
613 * This restriction comes from byte sized index implementation.
614 * Page size is normally 2^12 bytes and, in this case, if we want to use
615 * byte sized index which can represent 2^8 entries, the size of the object
616 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
617 * If minimum size of kmalloc is less than 16, we use it as minimum object
618 * size and give up to use byte sized index.
619 */
620#define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
621 (KMALLOC_MIN_SIZE) : 16)
622
623#ifdef CONFIG_RANDOM_KMALLOC_CACHES
624#define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies
625#else
626#define RANDOM_KMALLOC_CACHES_NR 0
627#endif
628
629/*
630 * Whenever changing this, take care of that kmalloc_type() and
631 * create_kmalloc_caches() still work as intended.
632 *
633 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
634 * is for accounted but unreclaimable and non-dma objects. All the other
635 * kmem caches can have both accounted and unaccounted objects.
636 */
637enum kmalloc_cache_type {
638 KMALLOC_NORMAL = 0,
639#ifndef CONFIG_ZONE_DMA
640 KMALLOC_DMA = KMALLOC_NORMAL,
641#endif
642#ifndef CONFIG_MEMCG
643 KMALLOC_CGROUP = KMALLOC_NORMAL,
644#endif
645 KMALLOC_RANDOM_START = KMALLOC_NORMAL,
646 KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
647#ifdef CONFIG_SLUB_TINY
648 KMALLOC_RECLAIM = KMALLOC_NORMAL,
649#else
650 KMALLOC_RECLAIM,
651#endif
652#ifdef CONFIG_ZONE_DMA
653 KMALLOC_DMA,
654#endif
655#ifdef CONFIG_MEMCG
656 KMALLOC_CGROUP,
657#endif
658 NR_KMALLOC_TYPES
659};
660
661typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];
662
663extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];
664
665/*
666 * Define gfp bits that should not be set for KMALLOC_NORMAL.
667 */
668#define KMALLOC_NOT_NORMAL_BITS \
669 (__GFP_RECLAIMABLE | \
670 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
671 (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0))
672
673extern unsigned long random_kmalloc_seed;
674
675static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller)
676{
677 /*
678 * The most common case is KMALLOC_NORMAL, so test for it
679 * with a single branch for all the relevant flags.
680 */
681 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
682#ifdef CONFIG_RANDOM_KMALLOC_CACHES
683 /* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
684 return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
685 ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
686#else
687 return KMALLOC_NORMAL;
688#endif
689
690 /*
691 * At least one of the flags has to be set. Their priorities in
692 * decreasing order are:
693 * 1) __GFP_DMA
694 * 2) __GFP_RECLAIMABLE
695 * 3) __GFP_ACCOUNT
696 */
697 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
698 return KMALLOC_DMA;
699 if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE))
700 return KMALLOC_RECLAIM;
701 else
702 return KMALLOC_CGROUP;
703}
704
705/*
706 * Figure out which kmalloc slab an allocation of a certain size
707 * belongs to.
708 * 0 = zero alloc
709 * 1 = 65 .. 96 bytes
710 * 2 = 129 .. 192 bytes
711 * n = 2^(n-1)+1 .. 2^n
712 *
713 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
714 * typical usage is via kmalloc_index() and therefore evaluated at compile-time.
715 * Callers where !size_is_constant should only be test modules, where runtime
716 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
717 */
718static __always_inline unsigned int __kmalloc_index(size_t size,
719 bool size_is_constant)
720{
721 if (!size)
722 return 0;
723
724 if (size <= KMALLOC_MIN_SIZE)
725 return KMALLOC_SHIFT_LOW;
726
727 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
728 return 1;
729 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
730 return 2;
731 if (size <= 8) return 3;
732 if (size <= 16) return 4;
733 if (size <= 32) return 5;
734 if (size <= 64) return 6;
735 if (size <= 128) return 7;
736 if (size <= 256) return 8;
737 if (size <= 512) return 9;
738 if (size <= 1024) return 10;
739 if (size <= 2 * 1024) return 11;
740 if (size <= 4 * 1024) return 12;
741 if (size <= 8 * 1024) return 13;
742 if (size <= 16 * 1024) return 14;
743 if (size <= 32 * 1024) return 15;
744 if (size <= 64 * 1024) return 16;
745 if (size <= 128 * 1024) return 17;
746 if (size <= 256 * 1024) return 18;
747 if (size <= 512 * 1024) return 19;
748 if (size <= 1024 * 1024) return 20;
749 if (size <= 2 * 1024 * 1024) return 21;
750
751 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
752 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
753 else
754 BUG();
755
756 /* Will never be reached. Needed because the compiler may complain */
757 return -1;
758}
759static_assert(PAGE_SHIFT <= 20);
760#define kmalloc_index(s) __kmalloc_index(s, true)
761
762#include <linux/alloc_tag.h>
763
764/**
765 * kmem_cache_alloc - Allocate an object
766 * @cachep: The cache to allocate from.
767 * @flags: See kmalloc().
768 *
769 * Allocate an object from this cache.
770 * See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
771 *
772 * Return: pointer to the new object or %NULL in case of error
773 */
774void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,
775 gfp_t flags) __assume_slab_alignment __malloc;
776#define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))
777
778void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
779 gfp_t gfpflags) __assume_slab_alignment __malloc;
780#define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))
781
782/**
783 * kmem_cache_charge - memcg charge an already allocated slab memory
784 * @objp: address of the slab object to memcg charge
785 * @gfpflags: describe the allocation context
786 *
787 * kmem_cache_charge allows charging a slab object to the current memcg,
788 * primarily in cases where charging at allocation time might not be possible
789 * because the target memcg is not known (i.e. softirq context)
790 *
791 * The objp should be pointer returned by the slab allocator functions like
792 * kmalloc (with __GFP_ACCOUNT in flags) or kmem_cache_alloc. The memcg charge
793 * behavior can be controlled through gfpflags parameter, which affects how the
794 * necessary internal metadata can be allocated. Including __GFP_NOFAIL denotes
795 * that overcharging is requested instead of failure, but is not applied for the
796 * internal metadata allocation.
797 *
798 * There are several cases where it will return true even if the charging was
799 * not done:
800 * More specifically:
801 *
802 * 1. For !CONFIG_MEMCG or cgroup_disable=memory systems.
803 * 2. Already charged slab objects.
804 * 3. For slab objects from KMALLOC_NORMAL caches - allocated by kmalloc()
805 * without __GFP_ACCOUNT
806 * 4. Allocating internal metadata has failed
807 *
808 * Return: true if charge was successful otherwise false.
809 */
810bool kmem_cache_charge(void *objp, gfp_t gfpflags);
811void kmem_cache_free(struct kmem_cache *s, void *objp);
812
813kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
814 unsigned int useroffset, unsigned int usersize,
815 void (*ctor)(void *));
816
817/*
818 * Bulk allocation and freeing operations. These are accelerated in an
819 * allocator specific way to avoid taking locks repeatedly or building
820 * metadata structures unnecessarily.
821 *
822 * Note that interrupts must be enabled when calling these functions.
823 */
824void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
825
826int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
827#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
828
829static __always_inline void kfree_bulk(size_t size, void **p)
830{
831 kmem_cache_free_bulk(NULL, size, p);
832}
833
834void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,
835 int node) __assume_slab_alignment __malloc;
836#define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))
837
838struct slab_sheaf *
839kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size);
840
841int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
842 struct slab_sheaf **sheafp, unsigned int size);
843
844void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
845 struct slab_sheaf *sheaf);
846
847void *kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *cachep, gfp_t gfp,
848 struct slab_sheaf *sheaf) __assume_slab_alignment __malloc;
849#define kmem_cache_alloc_from_sheaf(...) \
850 alloc_hooks(kmem_cache_alloc_from_sheaf_noprof(__VA_ARGS__))
851
852unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf);
853
854/*
855 * These macros allow declaring a kmem_buckets * parameter alongside size, which
856 * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call
857 * sites don't have to pass NULL.
858 */
859#ifdef CONFIG_SLAB_BUCKETS
860#define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b)
861#define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b)
862#define PASS_BUCKET_PARAM(_b) (_b)
863#else
864#define DECL_BUCKET_PARAMS(_size, _b) size_t (_size)
865#define PASS_BUCKET_PARAMS(_size, _b) (_size)
866#define PASS_BUCKET_PARAM(_b) NULL
867#endif
868
869/*
870 * The following functions are not to be used directly and are intended only
871 * for internal use from kmalloc() and kmalloc_node()
872 * with the exception of kunit tests
873 */
874
875void *__kmalloc_noprof(size_t size, gfp_t flags)
876 __assume_kmalloc_alignment __alloc_size(1);
877
878void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
879 __assume_kmalloc_alignment __alloc_size(1);
880
881void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
882 __assume_kmalloc_alignment __alloc_size(3);
883
884void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
885 int node, size_t size)
886 __assume_kmalloc_alignment __alloc_size(4);
887
888void *__kmalloc_large_noprof(size_t size, gfp_t flags)
889 __assume_page_alignment __alloc_size(1);
890
891void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
892 __assume_page_alignment __alloc_size(1);
893
894/**
895 * kmalloc - allocate kernel memory
896 * @size: how many bytes of memory are required.
897 * @flags: describe the allocation context
898 *
899 * kmalloc is the normal method of allocating memory
900 * for objects smaller than page size in the kernel.
901 *
902 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
903 * bytes. For @size of power of two bytes, the alignment is also guaranteed
904 * to be at least to the size. For other sizes, the alignment is guaranteed to
905 * be at least the largest power-of-two divisor of @size.
906 *
907 * The @flags argument may be one of the GFP flags defined at
908 * include/linux/gfp_types.h and described at
909 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
910 *
911 * The recommended usage of the @flags is described at
912 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
913 *
914 * Below is a brief outline of the most useful GFP flags
915 *
916 * %GFP_KERNEL
917 * Allocate normal kernel ram. May sleep.
918 *
919 * %GFP_NOWAIT
920 * Allocation will not sleep.
921 *
922 * %GFP_ATOMIC
923 * Allocation will not sleep. May use emergency pools.
924 *
925 * Also it is possible to set different flags by OR'ing
926 * in one or more of the following additional @flags:
927 *
928 * %__GFP_ZERO
929 * Zero the allocated memory before returning. Also see kzalloc().
930 *
931 * %__GFP_HIGH
932 * This allocation has high priority and may use emergency pools.
933 *
934 * %__GFP_NOFAIL
935 * Indicate that this allocation is in no way allowed to fail
936 * (think twice before using).
937 *
938 * %__GFP_NORETRY
939 * If memory is not immediately available,
940 * then give up at once.
941 *
942 * %__GFP_NOWARN
943 * If allocation fails, don't issue any warnings.
944 *
945 * %__GFP_RETRY_MAYFAIL
946 * Try really hard to succeed the allocation but fail
947 * eventually.
948 */
949static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)
950{
951 if (__builtin_constant_p(size) && size) {
952 unsigned int index;
953
954 if (size > KMALLOC_MAX_CACHE_SIZE)
955 return __kmalloc_large_noprof(size, flags);
956
957 index = kmalloc_index(size);
958 return __kmalloc_cache_noprof(
959 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
960 flags, size);
961 }
962 return __kmalloc_noprof(size, flags);
963}
964#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))
965
966void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node);
967#define kmalloc_nolock(...) alloc_hooks(kmalloc_nolock_noprof(__VA_ARGS__))
968
969/**
970 * __alloc_objs - Allocate objects of a given type using
971 * @KMALLOC: which size-based kmalloc wrapper to allocate with.
972 * @GFP: GFP flags for the allocation.
973 * @TYPE: type to allocate space for.
974 * @COUNT: how many @TYPE objects to allocate.
975 *
976 * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many
977 * allocated @TYPE objects, or NULL on failure.
978 */
979#define __alloc_objs(KMALLOC, GFP, TYPE, COUNT) \
980({ \
981 const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \
982 (TYPE *)KMALLOC(__obj_size, GFP); \
983})
984
985/**
986 * __alloc_flex - Allocate an object that has a trailing flexible array
987 * @KMALLOC: kmalloc wrapper function to use for allocation.
988 * @GFP: GFP flags for the allocation.
989 * @TYPE: type of structure to allocate space for.
990 * @FAM: The name of the flexible array member of @TYPE structure.
991 * @COUNT: how many @FAM elements to allocate space for.
992 *
993 * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing
994 * @FAM elements, or NULL on failure or if @COUNT cannot be represented
995 * by the member of @TYPE that counts the @FAM elements (annotated via
996 * __counted_by()).
997 */
998#define __alloc_flex(KMALLOC, GFP, TYPE, FAM, COUNT) \
999({ \
1000 const size_t __count = (COUNT); \
1001 const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \
1002 TYPE *__obj_ptr; \
1003 if (WARN_ON_ONCE(overflows_flex_counter_type(TYPE, FAM, __count))) \
1004 __obj_ptr = NULL; \
1005 else \
1006 __obj_ptr = KMALLOC(__obj_size, GFP); \
1007 if (__obj_ptr) \
1008 __set_flex_counter(__obj_ptr->FAM, __count); \
1009 __obj_ptr; \
1010})
1011
1012/**
1013 * kmalloc_obj - Allocate a single instance of the given type
1014 * @VAR_OR_TYPE: Variable or type to allocate.
1015 * @GFP: GFP flags for the allocation.
1016 *
1017 * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL
1018 * on failure.
1019 */
1020#define kmalloc_obj(VAR_OR_TYPE, GFP) \
1021 __alloc_objs(kmalloc, GFP, typeof(VAR_OR_TYPE), 1)
1022
1023/**
1024 * kmalloc_objs - Allocate an array of the given type
1025 * @VAR_OR_TYPE: Variable or type to allocate an array of.
1026 * @COUNT: How many elements in the array.
1027 * @GFP: GFP flags for the allocation.
1028 *
1029 * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success,
1030 * or NULL on failure.
1031 */
1032#define kmalloc_objs(VAR_OR_TYPE, COUNT, GFP) \
1033 __alloc_objs(kmalloc, GFP, typeof(VAR_OR_TYPE), COUNT)
1034
1035/**
1036 * kmalloc_flex - Allocate a single instance of the given flexible structure
1037 * @VAR_OR_TYPE: Variable or type to allocate (with its flex array).
1038 * @FAM: The name of the flexible array member of the structure.
1039 * @COUNT: How many flexible array member elements are desired.
1040 * @GFP: GFP flags for the allocation.
1041 *
1042 * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on
1043 * failure. If @FAM has been annotated with __counted_by(), the allocation
1044 * will immediately fail if @COUNT is larger than what the type of the
1045 * struct's counter variable can represent.
1046 */
1047#define kmalloc_flex(VAR_OR_TYPE, FAM, COUNT, GFP) \
1048 __alloc_flex(kmalloc, GFP, typeof(VAR_OR_TYPE), FAM, COUNT)
1049
1050/* All kzalloc aliases for kmalloc_(obj|objs|flex). */
1051#define kzalloc_obj(P, GFP) \
1052 __alloc_objs(kzalloc, GFP, typeof(P), 1)
1053#define kzalloc_objs(P, COUNT, GFP) \
1054 __alloc_objs(kzalloc, GFP, typeof(P), COUNT)
1055#define kzalloc_flex(P, FAM, COUNT, GFP) \
1056 __alloc_flex(kzalloc, GFP, typeof(P), FAM, COUNT)
1057
1058/* All kvmalloc aliases for kmalloc_(obj|objs|flex). */
1059#define kvmalloc_obj(P, GFP) \
1060 __alloc_objs(kvmalloc, GFP, typeof(P), 1)
1061#define kvmalloc_objs(P, COUNT, GFP) \
1062 __alloc_objs(kvmalloc, GFP, typeof(P), COUNT)
1063#define kvmalloc_flex(P, FAM, COUNT, GFP) \
1064 __alloc_flex(kvmalloc, GFP, typeof(P), FAM, COUNT)
1065
1066/* All kvzalloc aliases for kmalloc_(obj|objs|flex). */
1067#define kvzalloc_obj(P, GFP) \
1068 __alloc_objs(kvzalloc, GFP, typeof(P), 1)
1069#define kvzalloc_objs(P, COUNT, GFP) \
1070 __alloc_objs(kvzalloc, GFP, typeof(P), COUNT)
1071#define kvzalloc_flex(P, FAM, COUNT, GFP) \
1072 __alloc_flex(kvzalloc, GFP, typeof(P), FAM, COUNT)
1073
1074#define kmem_buckets_alloc(_b, _size, _flags) \
1075 alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
1076
1077#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
1078 alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_))
1079
1080static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
1081{
1082 if (__builtin_constant_p(size) && size) {
1083 unsigned int index;
1084
1085 if (size > KMALLOC_MAX_CACHE_SIZE)
1086 return __kmalloc_large_node_noprof(size, flags, node);
1087
1088 index = kmalloc_index(size);
1089 return __kmalloc_cache_node_noprof(
1090 kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
1091 flags, node, size);
1092 }
1093 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node);
1094}
1095#define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
1096
1097/**
1098 * kmalloc_array - allocate memory for an array.
1099 * @n: number of elements.
1100 * @size: element size.
1101 * @flags: the type of memory to allocate (see kmalloc).
1102 */
1103static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
1104{
1105 size_t bytes;
1106
1107 if (unlikely(check_mul_overflow(n, size, &bytes)))
1108 return NULL;
1109 return kmalloc_noprof(bytes, flags);
1110}
1111#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
1112
1113/**
1114 * krealloc_array - reallocate memory for an array.
1115 * @p: pointer to the memory chunk to reallocate
1116 * @new_n: new number of elements to alloc
1117 * @new_size: new size of a single member of the array
1118 * @flags: the type of memory to allocate (see kmalloc)
1119 *
1120 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
1121 * initial memory allocation, every subsequent call to this API for the same
1122 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
1123 * __GFP_ZERO is not fully honored by this API.
1124 *
1125 * See krealloc_noprof() for further details.
1126 *
1127 * In any case, the contents of the object pointed to are preserved up to the
1128 * lesser of the new and old sizes.
1129 */
1130static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,
1131 size_t new_n,
1132 size_t new_size,
1133 gfp_t flags)
1134{
1135 size_t bytes;
1136
1137 if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
1138 return NULL;
1139
1140 return krealloc_noprof(p, bytes, flags);
1141}
1142#define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__))
1143
1144/**
1145 * kcalloc - allocate memory for an array. The memory is set to zero.
1146 * @n: number of elements.
1147 * @size: element size.
1148 * @flags: the type of memory to allocate (see kmalloc).
1149 */
1150#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
1151
1152void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
1153 unsigned long caller) __alloc_size(1);
1154#define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
1155 __kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller)
1156#define kmalloc_node_track_caller(...) \
1157 alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
1158
1159/*
1160 * kmalloc_track_caller is a special version of kmalloc that records the
1161 * calling function of the routine calling it for slab leak tracking instead
1162 * of just the calling function (confusing, eh?).
1163 * It's useful when the call to kmalloc comes from a widely-used standard
1164 * allocator where we care about the real place the memory allocation
1165 * request comes from.
1166 */
1167#define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)
1168
1169#define kmalloc_track_caller_noprof(...) \
1170 kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_)
1171
1172static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
1173 int node)
1174{
1175 size_t bytes;
1176
1177 if (unlikely(check_mul_overflow(n, size, &bytes)))
1178 return NULL;
1179 if (__builtin_constant_p(n) && __builtin_constant_p(size))
1180 return kmalloc_node_noprof(bytes, flags, node);
1181 return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node);
1182}
1183#define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
1184
1185#define kcalloc_node(_n, _size, _flags, _node) \
1186 kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)
1187
1188/*
1189 * Shortcuts
1190 */
1191#define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
1192
1193/**
1194 * kzalloc - allocate memory. The memory is set to zero.
1195 * @size: how many bytes of memory are required.
1196 * @flags: the type of memory to allocate (see kmalloc).
1197 */
1198static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
1199{
1200 return kmalloc_noprof(size, flags | __GFP_ZERO);
1201}
1202#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
1203#define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1204
1205void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
1206 gfp_t flags, int node) __alloc_size(1);
1207#define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \
1208 __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node)
1209#define kvmalloc_node_align(...) \
1210 alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__))
1211#define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n)
1212#define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE)
1213#define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO)
1214
1215#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
1216
1217#define kmem_buckets_valloc(_b, _size, _flags) \
1218 alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE))
1219
1220static inline __alloc_size(1, 2) void *
1221kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
1222{
1223 size_t bytes;
1224
1225 if (unlikely(check_mul_overflow(n, size, &bytes)))
1226 return NULL;
1227
1228 return kvmalloc_node_align_noprof(bytes, 1, flags, node);
1229}
1230
1231#define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1232#define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
1233#define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
1234
1235#define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
1236#define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
1237#define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
1238
1239void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align,
1240 gfp_t flags, int nid) __realloc_size(2);
1241#define kvrealloc_node_align(...) \
1242 alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__))
1243#define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n)
1244#define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE)
1245
1246extern void kvfree(const void *addr);
1247DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
1248
1249extern void kvfree_sensitive(const void *addr, size_t len);
1250
1251unsigned int kmem_cache_size(struct kmem_cache *s);
1252
1253#ifndef CONFIG_KVFREE_RCU_BATCHED
1254static inline void kvfree_rcu_barrier(void)
1255{
1256 rcu_barrier();
1257}
1258
1259static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
1260{
1261 rcu_barrier();
1262}
1263
1264static inline void kfree_rcu_scheduler_running(void) { }
1265#else
1266void kvfree_rcu_barrier(void);
1267
1268void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
1269
1270void kfree_rcu_scheduler_running(void);
1271#endif
1272
1273/**
1274 * kmalloc_size_roundup - Report allocation bucket size for the given size
1275 *
1276 * @size: Number of bytes to round up from.
1277 *
1278 * This returns the number of bytes that would be available in a kmalloc()
1279 * allocation of @size bytes. For example, a 126 byte request would be
1280 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
1281 * for the general-purpose kmalloc()-based allocations, and is not for the
1282 * pre-sized kmem_cache_alloc()-based allocations.)
1283 *
1284 * Use this to kmalloc() the full bucket size ahead of time instead of using
1285 * ksize() to query the size after an allocation.
1286 */
1287size_t kmalloc_size_roundup(size_t size);
1288
1289void __init kmem_cache_init_late(void);
1290void __init kvfree_rcu_init(void);
1291
1292#endif /* _LINUX_SLAB_H */