Merge branch 'slab/next' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux

+124 -32

include/linux/slab.h

··· 4 4 * (C) SGI 2006, Christoph Lameter 5 5 * Cleaned up and restructured to ease the addition of alternative 6 6 * implementations of SLAB allocators. 7 + * (C) Linux Foundation 2008-2013 8 + * Unified interface for all slab allocators 7 9 */ 8 10 9 11 #ifndef _LINUX_SLAB_H ··· 96 94 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 97 95 (unsigned long)ZERO_SIZE_PTR) 98 96 97 + #include <linux/kmemleak.h> 99 98 100 99 struct mem_cgroup; 101 100 /* ··· 292 289 } 293 290 #endif /* !CONFIG_SLOB */ 294 291 292 + void *__kmalloc(size_t size, gfp_t flags); 293 + void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags); 294 + 295 + #ifdef CONFIG_NUMA 296 + void *__kmalloc_node(size_t size, gfp_t flags, int node); 297 + void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 298 + #else 299 + static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) 300 + { 301 + return __kmalloc(size, flags); 302 + } 303 + 304 + static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) 305 + { 306 + return kmem_cache_alloc(s, flags); 307 + } 308 + #endif 309 + 310 + #ifdef CONFIG_TRACING 311 + extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t); 312 + 313 + #ifdef CONFIG_NUMA 314 + extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 315 + gfp_t gfpflags, 316 + int node, size_t size); 317 + #else 318 + static __always_inline void * 319 + kmem_cache_alloc_node_trace(struct kmem_cache *s, 320 + gfp_t gfpflags, 321 + int node, size_t size) 322 + { 323 + return kmem_cache_alloc_trace(s, gfpflags, size); 324 + } 325 + #endif /* CONFIG_NUMA */ 326 + 327 + #else /* CONFIG_TRACING */ 328 + static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, 329 + gfp_t flags, size_t size) 330 + { 331 + return kmem_cache_alloc(s, flags); 332 + } 333 + 334 + static __always_inline void * 335 + kmem_cache_alloc_node_trace(struct kmem_cache *s, 336 + gfp_t gfpflags, 337 + int node, size_t size) 338 + { 339 + return kmem_cache_alloc_node(s, gfpflags, node); 340 + } 341 + #endif /* CONFIG_TRACING */ 342 + 295 343 #ifdef CONFIG_SLAB 296 344 #include <linux/slab_def.h> 297 345 #endif ··· 351 297 #include <linux/slub_def.h> 352 298 #endif 353 299 354 - #ifdef CONFIG_SLOB 355 - #include <linux/slob_def.h> 300 + static __always_inline void * 301 + kmalloc_order(size_t size, gfp_t flags, unsigned int order) 302 + { 303 + void *ret; 304 + 305 + flags |= (__GFP_COMP | __GFP_KMEMCG); 306 + ret = (void *) __get_free_pages(flags, order); 307 + kmemleak_alloc(ret, size, 1, flags); 308 + return ret; 309 + } 310 + 311 + #ifdef CONFIG_TRACING 312 + extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); 313 + #else 314 + static __always_inline void * 315 + kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 316 + { 317 + return kmalloc_order(size, flags, order); 318 + } 356 319 #endif 320 + 321 + static __always_inline void *kmalloc_large(size_t size, gfp_t flags) 322 + { 323 + unsigned int order = get_order(size); 324 + return kmalloc_order_trace(size, flags, order); 325 + } 326 + 327 + /** 328 + * kmalloc - allocate memory 329 + * @size: how many bytes of memory are required. 330 + * @flags: the type of memory to allocate (see kcalloc). 331 + * 332 + * kmalloc is the normal method of allocating memory 333 + * for objects smaller than page size in the kernel. 334 + */ 335 + static __always_inline void *kmalloc(size_t size, gfp_t flags) 336 + { 337 + if (__builtin_constant_p(size)) { 338 + if (size > KMALLOC_MAX_CACHE_SIZE) 339 + return kmalloc_large(size, flags); 340 + #ifndef CONFIG_SLOB 341 + if (!(flags & GFP_DMA)) { 342 + int index = kmalloc_index(size); 343 + 344 + if (!index) 345 + return ZERO_SIZE_PTR; 346 + 347 + return kmem_cache_alloc_trace(kmalloc_caches[index], 348 + flags, size); 349 + } 350 + #endif 351 + } 352 + return __kmalloc(size, flags); 353 + } 357 354 358 355 /* 359 356 * Determine size used for the nth kmalloc cache. ··· 424 319 return 192; 425 320 #endif 426 321 return 0; 322 + } 323 + 324 + static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 325 + { 326 + #ifndef CONFIG_SLOB 327 + if (__builtin_constant_p(size) && 328 + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { 329 + int i = kmalloc_index(size); 330 + 331 + if (!i) 332 + return ZERO_SIZE_PTR; 333 + 334 + return kmem_cache_alloc_node_trace(kmalloc_caches[i], 335 + flags, node, size); 336 + } 337 + #endif 338 + return __kmalloc_node(size, flags, node); 427 339 } 428 340 429 341 /* ··· 572 450 { 573 451 return kmalloc_array(n, size, flags | __GFP_ZERO); 574 452 } 575 - 576 - #if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB) 577 - /** 578 - * kmalloc_node - allocate memory from a specific node 579 - * @size: how many bytes of memory are required. 580 - * @flags: the type of memory to allocate (see kmalloc). 581 - * @node: node to allocate from. 582 - * 583 - * kmalloc() for non-local nodes, used to allocate from a specific node 584 - * if available. Equivalent to kmalloc() in the non-NUMA single-node 585 - * case. 586 - */ 587 - static inline void *kmalloc_node(size_t size, gfp_t flags, int node) 588 - { 589 - return kmalloc(size, flags); 590 - } 591 - 592 - static inline void *__kmalloc_node(size_t size, gfp_t flags, int node) 593 - { 594 - return __kmalloc(size, flags); 595 - } 596 - 597 - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); 598 - 599 - static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep, 600 - gfp_t flags, int node) 601 - { 602 - return kmem_cache_alloc(cachep, flags); 603 - } 604 - #endif /* !CONFIG_NUMA && !CONFIG_SLOB */ 605 453 606 454 /* 607 455 * kmalloc_track_caller is a special version of kmalloc that records the

-106

include/linux/slab_def.h

··· 3 3 4 4 /* 5 5 * Definitions unique to the original Linux SLAB allocator. 6 - * 7 - * What we provide here is a way to optimize the frequent kmalloc 8 - * calls in the kernel by selecting the appropriate general cache 9 - * if kmalloc was called with a size that can be established at 10 - * compile time. 11 - */ 12 - 13 - #include <linux/init.h> 14 - #include <linux/compiler.h> 15 - 16 - /* 17 - * struct kmem_cache 18 - * 19 - * manages a cache. 20 6 */ 21 7 22 8 struct kmem_cache { ··· 87 101 * Do not add fields after array[] 88 102 */ 89 103 }; 90 - 91 - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); 92 - void *__kmalloc(size_t size, gfp_t flags); 93 - 94 - #ifdef CONFIG_TRACING 95 - extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t); 96 - #else 97 - static __always_inline void * 98 - kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) 99 - { 100 - return kmem_cache_alloc(cachep, flags); 101 - } 102 - #endif 103 - 104 - static __always_inline void *kmalloc(size_t size, gfp_t flags) 105 - { 106 - struct kmem_cache *cachep; 107 - void *ret; 108 - 109 - if (__builtin_constant_p(size)) { 110 - int i; 111 - 112 - if (!size) 113 - return ZERO_SIZE_PTR; 114 - 115 - if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) 116 - return NULL; 117 - 118 - i = kmalloc_index(size); 119 - 120 - #ifdef CONFIG_ZONE_DMA 121 - if (flags & GFP_DMA) 122 - cachep = kmalloc_dma_caches[i]; 123 - else 124 - #endif 125 - cachep = kmalloc_caches[i]; 126 - 127 - ret = kmem_cache_alloc_trace(cachep, flags, size); 128 - 129 - return ret; 130 - } 131 - return __kmalloc(size, flags); 132 - } 133 - 134 - #ifdef CONFIG_NUMA 135 - extern void *__kmalloc_node(size_t size, gfp_t flags, int node); 136 - extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 137 - 138 - #ifdef CONFIG_TRACING 139 - extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, 140 - gfp_t flags, 141 - int nodeid, 142 - size_t size); 143 - #else 144 - static __always_inline void * 145 - kmem_cache_alloc_node_trace(struct kmem_cache *cachep, 146 - gfp_t flags, 147 - int nodeid, 148 - size_t size) 149 - { 150 - return kmem_cache_alloc_node(cachep, flags, nodeid); 151 - } 152 - #endif 153 - 154 - static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 155 - { 156 - struct kmem_cache *cachep; 157 - 158 - if (__builtin_constant_p(size)) { 159 - int i; 160 - 161 - if (!size) 162 - return ZERO_SIZE_PTR; 163 - 164 - if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) 165 - return NULL; 166 - 167 - i = kmalloc_index(size); 168 - 169 - #ifdef CONFIG_ZONE_DMA 170 - if (flags & GFP_DMA) 171 - cachep = kmalloc_dma_caches[i]; 172 - else 173 - #endif 174 - cachep = kmalloc_caches[i]; 175 - 176 - return kmem_cache_alloc_node_trace(cachep, flags, node, size); 177 - } 178 - return __kmalloc_node(size, flags, node); 179 - } 180 - 181 - #endif /* CONFIG_NUMA */ 182 104 183 105 #endif /* _LINUX_SLAB_DEF_H */

-31

include/linux/slob_def.h

··· 1 - #ifndef __LINUX_SLOB_DEF_H 2 - #define __LINUX_SLOB_DEF_H 3 - 4 - #include <linux/numa.h> 5 - 6 - void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 7 - 8 - static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep, 9 - gfp_t flags) 10 - { 11 - return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE); 12 - } 13 - 14 - void *__kmalloc_node(size_t size, gfp_t flags, int node); 15 - 16 - static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 17 - { 18 - return __kmalloc_node(size, flags, node); 19 - } 20 - 21 - static __always_inline void *kmalloc(size_t size, gfp_t flags) 22 - { 23 - return __kmalloc_node(size, flags, NUMA_NO_NODE); 24 - } 25 - 26 - static __always_inline void *__kmalloc(size_t size, gfp_t flags) 27 - { 28 - return kmalloc(size, flags); 29 - } 30 - 31 - #endif /* __LINUX_SLOB_DEF_H */

-110

include/linux/slub_def.h

··· 6 6 * 7 7 * (C) 2007 SGI, Christoph Lameter 8 8 */ 9 - #include <linux/types.h> 10 - #include <linux/gfp.h> 11 - #include <linux/bug.h> 12 - #include <linux/workqueue.h> 13 9 #include <linux/kobject.h> 14 - 15 - #include <linux/kmemleak.h> 16 10 17 11 enum stat_item { 18 12 ALLOC_FASTPATH, /* Allocation from cpu slab */ ··· 97 103 #endif 98 104 struct kmem_cache_node *node[MAX_NUMNODES]; 99 105 }; 100 - 101 - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); 102 - void *__kmalloc(size_t size, gfp_t flags); 103 - 104 - static __always_inline void * 105 - kmalloc_order(size_t size, gfp_t flags, unsigned int order) 106 - { 107 - void *ret; 108 - 109 - flags |= (__GFP_COMP | __GFP_KMEMCG); 110 - ret = (void *) __get_free_pages(flags, order); 111 - kmemleak_alloc(ret, size, 1, flags); 112 - return ret; 113 - } 114 - 115 - /** 116 - * Calling this on allocated memory will check that the memory 117 - * is expected to be in use, and print warnings if not. 118 - */ 119 - #ifdef CONFIG_SLUB_DEBUG 120 - extern bool verify_mem_not_deleted(const void *x); 121 - #else 122 - static inline bool verify_mem_not_deleted(const void *x) 123 - { 124 - return true; 125 - } 126 - #endif 127 - 128 - #ifdef CONFIG_TRACING 129 - extern void * 130 - kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size); 131 - extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); 132 - #else 133 - static __always_inline void * 134 - kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) 135 - { 136 - return kmem_cache_alloc(s, gfpflags); 137 - } 138 - 139 - static __always_inline void * 140 - kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 141 - { 142 - return kmalloc_order(size, flags, order); 143 - } 144 - #endif 145 - 146 - static __always_inline void *kmalloc_large(size_t size, gfp_t flags) 147 - { 148 - unsigned int order = get_order(size); 149 - return kmalloc_order_trace(size, flags, order); 150 - } 151 - 152 - static __always_inline void *kmalloc(size_t size, gfp_t flags) 153 - { 154 - if (__builtin_constant_p(size)) { 155 - if (size > KMALLOC_MAX_CACHE_SIZE) 156 - return kmalloc_large(size, flags); 157 - 158 - if (!(flags & GFP_DMA)) { 159 - int index = kmalloc_index(size); 160 - 161 - if (!index) 162 - return ZERO_SIZE_PTR; 163 - 164 - return kmem_cache_alloc_trace(kmalloc_caches[index], 165 - flags, size); 166 - } 167 - } 168 - return __kmalloc(size, flags); 169 - } 170 - 171 - #ifdef CONFIG_NUMA 172 - void *__kmalloc_node(size_t size, gfp_t flags, int node); 173 - void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 174 - 175 - #ifdef CONFIG_TRACING 176 - extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 177 - gfp_t gfpflags, 178 - int node, size_t size); 179 - #else 180 - static __always_inline void * 181 - kmem_cache_alloc_node_trace(struct kmem_cache *s, 182 - gfp_t gfpflags, 183 - int node, size_t size) 184 - { 185 - return kmem_cache_alloc_node(s, gfpflags, node); 186 - } 187 - #endif 188 - 189 - static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 190 - { 191 - if (__builtin_constant_p(size) && 192 - size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { 193 - int index = kmalloc_index(size); 194 - 195 - if (!index) 196 - return ZERO_SIZE_PTR; 197 - 198 - return kmem_cache_alloc_node_trace(kmalloc_caches[index], 199 - flags, node, size); 200 - } 201 - return __kmalloc_node(size, flags, node); 202 - } 203 - #endif 204 106 205 107 #endif /* _LINUX_SLUB_DEF_H */

+1 -1

init/Kconfig

··· 1602 1602 1603 1603 config SLUB_CPU_PARTIAL 1604 1604 default y 1605 - depends on SLUB 1605 + depends on SLUB && SMP 1606 1606 bool "SLUB per cpu partial cache" 1607 1607 help 1608 1608 Per cpu partial caches accellerate objects allocation and freeing

+11 -1

mm/slab_common.c

··· 19 19 #include <asm/tlbflush.h> 20 20 #include <asm/page.h> 21 21 #include <linux/memcontrol.h> 22 + #include <trace/events/kmem.h> 22 23 23 24 #include "slab.h" 24 25 ··· 374 373 { 375 374 int index; 376 375 377 - if (size > KMALLOC_MAX_SIZE) { 376 + if (unlikely(size > KMALLOC_MAX_SIZE)) { 378 377 WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 379 378 return NULL; 380 379 } ··· 496 495 } 497 496 #endif /* !CONFIG_SLOB */ 498 497 498 + #ifdef CONFIG_TRACING 499 + void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 500 + { 501 + void *ret = kmalloc_order(size, flags, order); 502 + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 503 + return ret; 504 + } 505 + EXPORT_SYMBOL(kmalloc_order_trace); 506 + #endif 499 507 500 508 #ifdef CONFIG_SLABINFO 501 509

+24 -4

mm/slob.c

··· 462 462 return ret; 463 463 } 464 464 465 - void *__kmalloc_node(size_t size, gfp_t gfp, int node) 465 + void *__kmalloc(size_t size, gfp_t gfp) 466 466 { 467 - return __do_kmalloc_node(size, gfp, node, _RET_IP_); 467 + return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_); 468 468 } 469 - EXPORT_SYMBOL(__kmalloc_node); 469 + EXPORT_SYMBOL(__kmalloc); 470 470 471 471 #ifdef CONFIG_TRACING 472 472 void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller) ··· 534 534 return 0; 535 535 } 536 536 537 - void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node) 537 + void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node) 538 538 { 539 539 void *b; 540 540 ··· 560 560 kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags); 561 561 return b; 562 562 } 563 + EXPORT_SYMBOL(slob_alloc_node); 564 + 565 + void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 566 + { 567 + return slob_alloc_node(cachep, flags, NUMA_NO_NODE); 568 + } 569 + EXPORT_SYMBOL(kmem_cache_alloc); 570 + 571 + #ifdef CONFIG_NUMA 572 + void *__kmalloc_node(size_t size, gfp_t gfp, int node) 573 + { 574 + return __do_kmalloc_node(size, gfp, node, _RET_IP_); 575 + } 576 + EXPORT_SYMBOL(__kmalloc_node); 577 + 578 + void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node) 579 + { 580 + return slob_alloc_node(cachep, gfp, node); 581 + } 563 582 EXPORT_SYMBOL(kmem_cache_alloc_node); 583 + #endif 564 584 565 585 static void __kmem_cache_free(void *b, int size) 566 586 {

+56 -86

mm/slub.c

··· 373 373 #endif 374 374 { 375 375 slab_lock(page); 376 - if (page->freelist == freelist_old && page->counters == counters_old) { 376 + if (page->freelist == freelist_old && 377 + page->counters == counters_old) { 377 378 page->freelist = freelist_new; 378 379 page->counters = counters_new; 379 380 slab_unlock(page); ··· 412 411 413 412 local_irq_save(flags); 414 413 slab_lock(page); 415 - if (page->freelist == freelist_old && page->counters == counters_old) { 414 + if (page->freelist == freelist_old && 415 + page->counters == counters_old) { 416 416 page->freelist = freelist_new; 417 417 page->counters = counters_new; 418 418 slab_unlock(page); ··· 555 553 556 554 static void print_page_info(struct page *page) 557 555 { 558 - printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", 559 - page, page->objects, page->inuse, page->freelist, page->flags); 556 + printk(KERN_ERR 557 + "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", 558 + page, page->objects, page->inuse, page->freelist, page->flags); 560 559 561 560 } 562 561 ··· 632 629 print_trailer(s, page, object); 633 630 } 634 631 635 - static void slab_err(struct kmem_cache *s, struct page *page, const char *fmt, ...) 632 + static void slab_err(struct kmem_cache *s, struct page *page, 633 + const char *fmt, ...) 636 634 { 637 635 va_list args; 638 636 char buf[100]; ··· 792 788 } else { 793 789 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 794 790 check_bytes_and_report(s, page, p, "Alignment padding", 795 - endobject, POISON_INUSE, s->inuse - s->object_size); 791 + endobject, POISON_INUSE, 792 + s->inuse - s->object_size); 796 793 } 797 794 } 798 795 ··· 878 873 object_err(s, page, object, 879 874 "Freechain corrupt"); 880 875 set_freepointer(s, object, NULL); 881 - break; 882 876 } else { 883 877 slab_err(s, page, "Freepointer corrupt"); 884 878 page->freelist = NULL; ··· 922 918 page->freelist); 923 919 924 920 if (!alloc) 925 - print_section("Object ", (void *)object, s->object_size); 921 + print_section("Object ", (void *)object, 922 + s->object_size); 926 923 927 924 dump_stack(); 928 925 } ··· 942 937 return should_failslab(s->object_size, flags, s->flags); 943 938 } 944 939 945 - static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object) 940 + static inline void slab_post_alloc_hook(struct kmem_cache *s, 941 + gfp_t flags, void *object) 946 942 { 947 943 flags &= gfp_allowed_mask; 948 944 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); ··· 1045 1039 init_tracking(s, object); 1046 1040 } 1047 1041 1048 - static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *page, 1042 + static noinline int alloc_debug_processing(struct kmem_cache *s, 1043 + struct page *page, 1049 1044 void *object, unsigned long addr) 1050 1045 { 1051 1046 if (!check_slab(s, page)) ··· 1750 1743 /* 1751 1744 * Remove the cpu slab 1752 1745 */ 1753 - static void deactivate_slab(struct kmem_cache *s, struct page *page, void *freelist) 1746 + static void deactivate_slab(struct kmem_cache *s, struct page *page, 1747 + void *freelist) 1754 1748 { 1755 1749 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; 1756 1750 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ··· 2007 1999 page->pobjects = pobjects; 2008 2000 page->next = oldpage; 2009 2001 2010 - } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage); 2002 + } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) 2003 + != oldpage); 2011 2004 #endif 2012 2005 } 2013 2006 ··· 2178 2169 } 2179 2170 2180 2171 /* 2181 - * Check the page->freelist of a page and either transfer the freelist to the per cpu freelist 2182 - * or deactivate the page. 2172 + * Check the page->freelist of a page and either transfer the freelist to the 2173 + * per cpu freelist or deactivate the page. 2183 2174 * 2184 2175 * The page is still frozen if the return value is not NULL. 2185 2176 * ··· 2323 2314 goto load_freelist; 2324 2315 2325 2316 /* Only entered in the debug case */ 2326 - if (kmem_cache_debug(s) && !alloc_debug_processing(s, page, freelist, addr)) 2317 + if (kmem_cache_debug(s) && 2318 + !alloc_debug_processing(s, page, freelist, addr)) 2327 2319 goto new_slab; /* Slab failed checks. Next slab needed */ 2328 2320 2329 2321 deactivate_slab(s, page, get_freepointer(s, freelist)); ··· 2382 2372 2383 2373 object = c->freelist; 2384 2374 page = c->page; 2385 - if (unlikely(!object || !page || !node_match(page, node))) 2375 + if (unlikely(!object || !node_match(page, node))) 2386 2376 object = __slab_alloc(s, gfpflags, node, addr, c); 2387 2377 2388 2378 else { ··· 2392 2382 * The cmpxchg will only match if there was no additional 2393 2383 * operation and if we are on the right processor. 2394 2384 * 2395 - * The cmpxchg does the following atomically (without lock semantics!) 2385 + * The cmpxchg does the following atomically (without lock 2386 + * semantics!) 2396 2387 * 1. Relocate first pointer to the current per cpu area. 2397 2388 * 2. Verify that tid and freelist have not been changed 2398 2389 * 3. If they were not changed replace tid and freelist 2399 2390 * 2400 - * Since this is without lock semantics the protection is only against 2401 - * code executing on this cpu *not* from access by other cpus. 2391 + * Since this is without lock semantics the protection is only 2392 + * against code executing on this cpu *not* from access by 2393 + * other cpus. 2402 2394 */ 2403 2395 if (unlikely(!this_cpu_cmpxchg_double( 2404 2396 s->cpu_slab->freelist, s->cpu_slab->tid, ··· 2432 2420 { 2433 2421 void *ret = slab_alloc(s, gfpflags, _RET_IP_); 2434 2422 2435 - trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags); 2423 + trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, 2424 + s->size, gfpflags); 2436 2425 2437 2426 return ret; 2438 2427 } ··· 2447 2434 return ret; 2448 2435 } 2449 2436 EXPORT_SYMBOL(kmem_cache_alloc_trace); 2450 - 2451 - void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 2452 - { 2453 - void *ret = kmalloc_order(size, flags, order); 2454 - trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 2455 - return ret; 2456 - } 2457 - EXPORT_SYMBOL(kmalloc_order_trace); 2458 2437 #endif 2459 2438 2460 2439 #ifdef CONFIG_NUMA ··· 2517 2512 if (kmem_cache_has_cpu_partial(s) && !prior) 2518 2513 2519 2514 /* 2520 - * Slab was on no list before and will be partially empty 2521 - * We can defer the list move and instead freeze it. 2515 + * Slab was on no list before and will be 2516 + * partially empty 2517 + * We can defer the list move and instead 2518 + * freeze it. 2522 2519 */ 2523 2520 new.frozen = 1; 2524 2521 ··· 3078 3071 * A) The number of objects from per cpu partial slabs dumped to the 3079 3072 * per node list when we reach the limit. 3080 3073 * B) The number of objects in cpu partial slabs to extract from the 3081 - * per node list when we run out of per cpu objects. We only fetch 50% 3082 - * to keep some capacity around for frees. 3074 + * per node list when we run out of per cpu objects. We only fetch 3075 + * 50% to keep some capacity around for frees. 3083 3076 */ 3084 3077 if (!kmem_cache_has_cpu_partial(s)) 3085 3078 s->cpu_partial = 0; ··· 3106 3099 if (flags & SLAB_PANIC) 3107 3100 panic("Cannot create slab %s size=%lu realsize=%u " 3108 3101 "order=%u offset=%u flags=%lx\n", 3109 - s->name, (unsigned long)s->size, s->size, oo_order(s->oo), 3110 - s->offset, flags); 3102 + s->name, (unsigned long)s->size, s->size, 3103 + oo_order(s->oo), s->offset, flags); 3111 3104 return -EINVAL; 3112 3105 } 3113 3106 ··· 3322 3315 return slab_ksize(page->slab_cache); 3323 3316 } 3324 3317 EXPORT_SYMBOL(ksize); 3325 - 3326 - #ifdef CONFIG_SLUB_DEBUG 3327 - bool verify_mem_not_deleted(const void *x) 3328 - { 3329 - struct page *page; 3330 - void *object = (void *)x; 3331 - unsigned long flags; 3332 - bool rv; 3333 - 3334 - if (unlikely(ZERO_OR_NULL_PTR(x))) 3335 - return false; 3336 - 3337 - local_irq_save(flags); 3338 - 3339 - page = virt_to_head_page(x); 3340 - if (unlikely(!PageSlab(page))) { 3341 - /* maybe it was from stack? */ 3342 - rv = true; 3343 - goto out_unlock; 3344 - } 3345 - 3346 - slab_lock(page); 3347 - if (on_freelist(page->slab_cache, page, object)) { 3348 - object_err(page->slab_cache, page, object, "Object is on free-list"); 3349 - rv = false; 3350 - } else { 3351 - rv = true; 3352 - } 3353 - slab_unlock(page); 3354 - 3355 - out_unlock: 3356 - local_irq_restore(flags); 3357 - return rv; 3358 - } 3359 - EXPORT_SYMBOL(verify_mem_not_deleted); 3360 - #endif 3361 3318 3362 3319 void kfree(const void *x) 3363 3320 { ··· 4133 4162 !cpumask_empty(to_cpumask(l->cpus)) && 4134 4163 len < PAGE_SIZE - 60) { 4135 4164 len += sprintf(buf + len, " cpus="); 4136 - len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50, 4165 + len += cpulist_scnprintf(buf + len, 4166 + PAGE_SIZE - len - 50, 4137 4167 to_cpumask(l->cpus)); 4138 4168 } 4139 4169 4140 4170 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && 4141 4171 len < PAGE_SIZE - 60) { 4142 4172 len += sprintf(buf + len, " nodes="); 4143 - len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50, 4144 - l->nodes); 4173 + len += nodelist_scnprintf(buf + len, 4174 + PAGE_SIZE - len - 50, 4175 + l->nodes); 4145 4176 } 4146 4177 4147 4178 len += sprintf(buf + len, "\n"); ··· 4241 4268 int node; 4242 4269 int x; 4243 4270 unsigned long *nodes; 4244 - unsigned long *per_cpu; 4245 4271 4246 - nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); 4272 + nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); 4247 4273 if (!nodes) 4248 4274 return -ENOMEM; 4249 - per_cpu = nodes + nr_node_ids; 4250 4275 4251 4276 if (flags & SO_CPU) { 4252 4277 int cpu; 4253 4278 4254 4279 for_each_possible_cpu(cpu) { 4255 - struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 4280 + struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 4281 + cpu); 4256 4282 int node; 4257 4283 struct page *page; 4258 4284 ··· 4276 4304 total += x; 4277 4305 nodes[node] += x; 4278 4306 } 4279 - 4280 - per_cpu[node]++; 4281 4307 } 4282 4308 } 4283 4309 ··· 4285 4315 for_each_node_state(node, N_NORMAL_MEMORY) { 4286 4316 struct kmem_cache_node *n = get_node(s, node); 4287 4317 4288 - if (flags & SO_TOTAL) 4289 - x = atomic_long_read(&n->total_objects); 4290 - else if (flags & SO_OBJECTS) 4291 - x = atomic_long_read(&n->total_objects) - 4292 - count_partial(n, count_free); 4293 - 4318 + if (flags & SO_TOTAL) 4319 + x = atomic_long_read(&n->total_objects); 4320 + else if (flags & SO_OBJECTS) 4321 + x = atomic_long_read(&n->total_objects) - 4322 + count_partial(n, count_free); 4294 4323 else 4295 4324 x = atomic_long_read(&n->nr_slabs); 4296 4325 total += x; ··· 5105 5136 5106 5137 #ifdef CONFIG_MEMCG_KMEM 5107 5138 if (!is_root_cache(s)) 5108 - p += sprintf(p, "-%08d", memcg_cache_id(s->memcg_params->memcg)); 5139 + p += sprintf(p, "-%08d", 5140 + memcg_cache_id(s->memcg_params->memcg)); 5109 5141 #endif 5110 5142 5111 5143 BUG_ON(p > name + ID_STR_LENGTH - 1);

Configure Feed

Configure Feed