Merge branch 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux

+42 -15

include/linux/slab.h

··· 169 169 struct list_head list; /* List of all slab caches on the system */ 170 170 }; 171 171 172 - #define KMALLOC_MAX_SIZE (1UL << 30) 173 - 174 - #include <linux/slob_def.h> 175 - 176 - #else /* CONFIG_SLOB */ 172 + #endif /* CONFIG_SLOB */ 177 173 178 174 /* 179 175 * Kmalloc array related definitions ··· 191 195 #ifndef KMALLOC_SHIFT_LOW 192 196 #define KMALLOC_SHIFT_LOW 5 193 197 #endif 194 - #else 198 + #endif 199 + 200 + #ifdef CONFIG_SLUB 195 201 /* 196 202 * SLUB allocates up to order 2 pages directly and otherwise 197 203 * passes the request to the page allocator. 198 204 */ 199 205 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 200 206 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) 207 + #ifndef KMALLOC_SHIFT_LOW 208 + #define KMALLOC_SHIFT_LOW 3 209 + #endif 210 + #endif 211 + 212 + #ifdef CONFIG_SLOB 213 + /* 214 + * SLOB passes all page size and larger requests to the page allocator. 215 + * No kmalloc array is necessary since objects of different sizes can 216 + * be allocated from the same page. 217 + */ 218 + #define KMALLOC_SHIFT_MAX 30 219 + #define KMALLOC_SHIFT_HIGH PAGE_SHIFT 201 220 #ifndef KMALLOC_SHIFT_LOW 202 221 #define KMALLOC_SHIFT_LOW 3 203 222 #endif ··· 232 221 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 233 222 #endif 234 223 224 + #ifndef CONFIG_SLOB 235 225 extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; 236 226 #ifdef CONFIG_ZONE_DMA 237 227 extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; ··· 287 275 /* Will never be reached. Needed because the compiler may complain */ 288 276 return -1; 289 277 } 278 + #endif /* !CONFIG_SLOB */ 290 279 291 280 #ifdef CONFIG_SLAB 292 281 #include <linux/slab_def.h> 293 - #elif defined(CONFIG_SLUB) 282 + #endif 283 + 284 + #ifdef CONFIG_SLUB 294 285 #include <linux/slub_def.h> 295 - #else 296 - #error "Unknown slab allocator" 286 + #endif 287 + 288 + #ifdef CONFIG_SLOB 289 + #include <linux/slob_def.h> 297 290 #endif 298 291 299 292 /* ··· 308 291 */ 309 292 static __always_inline int kmalloc_size(int n) 310 293 { 294 + #ifndef CONFIG_SLOB 311 295 if (n > 2) 312 296 return 1 << n; 313 297 ··· 317 299 318 300 if (n == 2 && KMALLOC_MIN_SIZE <= 64) 319 301 return 192; 320 - 302 + #endif 321 303 return 0; 322 304 } 323 - #endif /* !CONFIG_SLOB */ 324 305 325 306 /* 326 307 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. ··· 373 356 void print_slabinfo_header(struct seq_file *m); 374 357 375 358 /** 376 - * kmalloc_array - allocate memory for an array. 377 - * @n: number of elements. 378 - * @size: element size. 359 + * kmalloc - allocate memory 360 + * @size: how many bytes of memory are required. 379 361 * @flags: the type of memory to allocate. 380 362 * 381 363 * The @flags argument may be one of: ··· 421 405 * There are other flags available as well, but these are not intended 422 406 * for general use, and so are not documented here. For a full list of 423 407 * potential flags, always refer to linux/gfp.h. 408 + * 409 + * kmalloc is the normal method of allocating memory 410 + * in the kernel. 411 + */ 412 + static __always_inline void *kmalloc(size_t size, gfp_t flags); 413 + 414 + /** 415 + * kmalloc_array - allocate memory for an array. 416 + * @n: number of elements. 417 + * @size: element size. 418 + * @flags: the type of memory to allocate (see kmalloc). 424 419 */ 425 420 static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags) 426 421 { ··· 455 428 /** 456 429 * kmalloc_node - allocate memory from a specific node 457 430 * @size: how many bytes of memory are required. 458 - * @flags: the type of memory to allocate (see kcalloc). 431 + * @flags: the type of memory to allocate (see kmalloc). 459 432 * @node: node to allocate from. 460 433 * 461 434 * kmalloc() for non-local nodes, used to allocate from a specific node

-8

include/linux/slob_def.h

··· 18 18 return __kmalloc_node(size, flags, node); 19 19 } 20 20 21 - /** 22 - * kmalloc - allocate memory 23 - * @size: how many bytes of memory are required. 24 - * @flags: the type of memory to allocate (see kcalloc). 25 - * 26 - * kmalloc is the normal method of allocating memory 27 - * in the kernel. 28 - */ 29 21 static __always_inline void *kmalloc(size_t size, gfp_t flags) 30 22 { 31 23 return __kmalloc_node(size, flags, NUMA_NO_NODE);

+11

init/Kconfig

··· 1596 1596 1597 1597 endchoice 1598 1598 1599 + config SLUB_CPU_PARTIAL 1600 + default y 1601 + depends on SLUB 1602 + bool "SLUB per cpu partial cache" 1603 + help 1604 + Per cpu partial caches accellerate objects allocation and freeing 1605 + that is local to a processor at the price of more indeterminism 1606 + in the latency of the free. On overflow these caches will be cleared 1607 + which requires the taking of locks that may cause latency spikes. 1608 + Typically one would choose no for a realtime system. 1609 + 1599 1610 config MMAP_ALLOW_UNINITIALIZED 1600 1611 bool "Allow mmapped anonymous memory to be uninitialized" 1601 1612 depends on EXPERT && !MMU

+23 -28

mm/slab.c

··· 565 565 if (slab_state < UP) 566 566 return; 567 567 568 - for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { 568 + for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) { 569 569 struct kmem_cache_node *n; 570 570 struct kmem_cache *cache = kmalloc_caches[i]; 571 571 ··· 1180 1180 return 0; 1181 1181 } 1182 1182 1183 + static inline int slabs_tofree(struct kmem_cache *cachep, 1184 + struct kmem_cache_node *n) 1185 + { 1186 + return (n->free_objects + cachep->num - 1) / cachep->num; 1187 + } 1188 + 1183 1189 static void __cpuinit cpuup_canceled(long cpu) 1184 1190 { 1185 1191 struct kmem_cache *cachep; ··· 1247 1241 n = cachep->node[node]; 1248 1242 if (!n) 1249 1243 continue; 1250 - drain_freelist(cachep, n, n->free_objects); 1244 + drain_freelist(cachep, n, slabs_tofree(cachep, n)); 1251 1245 } 1252 1246 } 1253 1247 ··· 1414 1408 if (!n) 1415 1409 continue; 1416 1410 1417 - drain_freelist(cachep, n, n->free_objects); 1411 + drain_freelist(cachep, n, slabs_tofree(cachep, n)); 1418 1412 1419 1413 if (!list_empty(&n->slabs_full) || 1420 1414 !list_empty(&n->slabs_partial)) { ··· 2538 2532 if (!n) 2539 2533 continue; 2540 2534 2541 - drain_freelist(cachep, n, n->free_objects); 2535 + drain_freelist(cachep, n, slabs_tofree(cachep, n)); 2542 2536 2543 2537 ret += !list_empty(&n->slabs_full) || 2544 2538 !list_empty(&n->slabs_partial); ··· 3344 3338 return obj; 3345 3339 } 3346 3340 3347 - /** 3348 - * kmem_cache_alloc_node - Allocate an object on the specified node 3349 - * @cachep: The cache to allocate from. 3350 - * @flags: See kmalloc(). 3351 - * @nodeid: node number of the target node. 3352 - * @caller: return address of caller, used for debug information 3353 - * 3354 - * Identical to kmem_cache_alloc but it will allocate memory on the given 3355 - * node, which can improve the performance for cpu bound structures. 3356 - * 3357 - * Fallback to other node is possible if __GFP_THISNODE is not set. 3358 - */ 3359 3341 static __always_inline void * 3360 3342 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, 3361 3343 unsigned long caller) ··· 3637 3643 #endif 3638 3644 3639 3645 #ifdef CONFIG_NUMA 3646 + /** 3647 + * kmem_cache_alloc_node - Allocate an object on the specified node 3648 + * @cachep: The cache to allocate from. 3649 + * @flags: See kmalloc(). 3650 + * @nodeid: node number of the target node. 3651 + * 3652 + * Identical to kmem_cache_alloc but it will allocate memory on the given 3653 + * node, which can improve the performance for cpu bound structures. 3654 + * 3655 + * Fallback to other node is possible if __GFP_THISNODE is not set. 3656 + */ 3640 3657 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 3641 3658 { 3642 3659 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); ··· 4436 4431 return 0; 4437 4432 } 4438 4433 4439 - static void *s_next(struct seq_file *m, void *p, loff_t *pos) 4440 - { 4441 - return seq_list_next(p, &slab_caches, pos); 4442 - } 4443 - 4444 - static void s_stop(struct seq_file *m, void *p) 4445 - { 4446 - mutex_unlock(&slab_mutex); 4447 - } 4448 - 4449 4434 static const struct seq_operations slabstats_op = { 4450 4435 .start = leaks_start, 4451 - .next = s_next, 4452 - .stop = s_stop, 4436 + .next = slab_next, 4437 + .stop = slab_stop, 4453 4438 .show = leaks_show, 4454 4439 }; 4455 4440

+3

mm/slab.h

··· 271 271 #endif 272 272 273 273 }; 274 + 275 + void *slab_next(struct seq_file *m, void *p, loff_t *pos); 276 + void slab_stop(struct seq_file *m, void *p);

+13 -5

mm/slab_common.c

··· 497 497 498 498 499 499 #ifdef CONFIG_SLABINFO 500 + 501 + #ifdef CONFIG_SLAB 502 + #define SLABINFO_RIGHTS (S_IWUSR | S_IRUSR) 503 + #else 504 + #define SLABINFO_RIGHTS S_IRUSR 505 + #endif 506 + 500 507 void print_slabinfo_header(struct seq_file *m) 501 508 { 502 509 /* ··· 538 531 return seq_list_start(&slab_caches, *pos); 539 532 } 540 533 541 - static void *s_next(struct seq_file *m, void *p, loff_t *pos) 534 + void *slab_next(struct seq_file *m, void *p, loff_t *pos) 542 535 { 543 536 return seq_list_next(p, &slab_caches, pos); 544 537 } 545 538 546 - static void s_stop(struct seq_file *m, void *p) 539 + void slab_stop(struct seq_file *m, void *p) 547 540 { 548 541 mutex_unlock(&slab_mutex); 549 542 } ··· 620 613 */ 621 614 static const struct seq_operations slabinfo_op = { 622 615 .start = s_start, 623 - .next = s_next, 624 - .stop = s_stop, 616 + .next = slab_next, 617 + .stop = slab_stop, 625 618 .show = s_show, 626 619 }; 627 620 ··· 640 633 641 634 static int __init slab_proc_init(void) 642 635 { 643 - proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations); 636 + proc_create("slabinfo", SLABINFO_RIGHTS, NULL, 637 + &proc_slabinfo_operations); 644 638 return 0; 645 639 } 646 640 module_init(slab_proc_init);

+2 -2

mm/slob.c

··· 122 122 } 123 123 124 124 #define SLOB_UNIT sizeof(slob_t) 125 - #define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT) 125 + #define SLOB_UNITS(size) DIV_ROUND_UP(size, SLOB_UNIT) 126 126 127 127 /* 128 128 * struct slob_rcu is inserted at the tail of allocated slob blocks, which ··· 554 554 flags, node); 555 555 } 556 556 557 - if (c->ctor) 557 + if (b && c->ctor) 558 558 c->ctor(b); 559 559 560 560 kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);

+27 -11

mm/slub.c

··· 123 123 #endif 124 124 } 125 125 126 + static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 127 + { 128 + #ifdef CONFIG_SLUB_CPU_PARTIAL 129 + return !kmem_cache_debug(s); 130 + #else 131 + return false; 132 + #endif 133 + } 134 + 126 135 /* 127 136 * Issues still to be resolved: 128 137 * ··· 1582 1573 put_cpu_partial(s, page, 0); 1583 1574 stat(s, CPU_PARTIAL_NODE); 1584 1575 } 1585 - if (kmem_cache_debug(s) || available > s->cpu_partial / 2) 1576 + if (!kmem_cache_has_cpu_partial(s) 1577 + || available > s->cpu_partial / 2) 1586 1578 break; 1587 1579 1588 1580 } ··· 1894 1884 static void unfreeze_partials(struct kmem_cache *s, 1895 1885 struct kmem_cache_cpu *c) 1896 1886 { 1887 + #ifdef CONFIG_SLUB_CPU_PARTIAL 1897 1888 struct kmem_cache_node *n = NULL, *n2 = NULL; 1898 1889 struct page *page, *discard_page = NULL; 1899 1890 ··· 1949 1938 discard_slab(s, page); 1950 1939 stat(s, FREE_SLAB); 1951 1940 } 1941 + #endif 1952 1942 } 1953 1943 1954 1944 /* ··· 1963 1951 */ 1964 1952 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) 1965 1953 { 1954 + #ifdef CONFIG_SLUB_CPU_PARTIAL 1966 1955 struct page *oldpage; 1967 1956 int pages; 1968 1957 int pobjects; 1958 + 1959 + if (!s->cpu_partial) 1960 + return; 1969 1961 1970 1962 do { 1971 1963 pages = 0; ··· 2003 1987 page->next = oldpage; 2004 1988 2005 1989 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage); 1990 + #endif 2006 1991 } 2007 1992 2008 1993 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) ··· 2375 2358 2376 2359 object = c->freelist; 2377 2360 page = c->page; 2378 - if (unlikely(!object || !node_match(page, node))) 2361 + if (unlikely(!object || !page || !node_match(page, node))) 2379 2362 object = __slab_alloc(s, gfpflags, node, addr, c); 2380 2363 2381 2364 else { ··· 2512 2495 new.inuse--; 2513 2496 if ((!new.inuse || !prior) && !was_frozen) { 2514 2497 2515 - if (!kmem_cache_debug(s) && !prior) 2498 + if (kmem_cache_has_cpu_partial(s) && !prior) 2516 2499 2517 2500 /* 2518 2501 * Slab was on no list before and will be partially empty ··· 2567 2550 * Objects left in the slab. If it was not on the partial list before 2568 2551 * then add it. 2569 2552 */ 2570 - if (kmem_cache_debug(s) && unlikely(!prior)) { 2571 - remove_full(s, page); 2553 + if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 2554 + if (kmem_cache_debug(s)) 2555 + remove_full(s, page); 2572 2556 add_partial(n, page, DEACTIVATE_TO_TAIL); 2573 2557 stat(s, FREE_ADD_PARTIAL); 2574 2558 } ··· 3077 3059 * per node list when we run out of per cpu objects. We only fetch 50% 3078 3060 * to keep some capacity around for frees. 3079 3061 */ 3080 - if (kmem_cache_debug(s)) 3062 + if (!kmem_cache_has_cpu_partial(s)) 3081 3063 s->cpu_partial = 0; 3082 3064 else if (s->size >= PAGE_SIZE) 3083 3065 s->cpu_partial = 2; ··· 4474 4456 err = strict_strtoul(buf, 10, &objects); 4475 4457 if (err) 4476 4458 return err; 4477 - if (objects && kmem_cache_debug(s)) 4459 + if (objects && !kmem_cache_has_cpu_partial(s)) 4478 4460 return -EINVAL; 4479 4461 4480 4462 s->cpu_partial = objects; ··· 5287 5269 #ifdef CONFIG_SLABINFO 5288 5270 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 5289 5271 { 5290 - unsigned long nr_partials = 0; 5291 5272 unsigned long nr_slabs = 0; 5292 5273 unsigned long nr_objs = 0; 5293 5274 unsigned long nr_free = 0; ··· 5298 5281 if (!n) 5299 5282 continue; 5300 5283 5301 - nr_partials += n->nr_partial; 5302 - nr_slabs += atomic_long_read(&n->nr_slabs); 5303 - nr_objs += atomic_long_read(&n->total_objects); 5284 + nr_slabs += node_nr_slabs(n); 5285 + nr_objs += node_nr_objs(n); 5304 5286 nr_free += count_partial(n, count_free); 5305 5287 } 5306 5288

Configure Feed

Configure Feed