tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

zsmalloc: make common caches global

Currently, zsmalloc creates kmem_cache of handles and zspages for each
pool, which may be suboptimal from the memory usage point of view (extra
internal fragmentation per pool). Systems that create multiple zsmalloc
pools may benefit from shared common zsmalloc caches.

Make handles and zspages kmem caches global. The memory savings depend on
particular setup and data patterns and can be found via slabinfo.

Link: https://lkml.kernel.org/r/20260117025406.799428-1-senozhatsky@chromium.org
Signed-off-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Reviewed-by: Yosry Ahmed <yosry.ahmed@linux.dev>
Cc: Brian Geffon <bgeffon@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Sergey Senozhatsky and committed by
Andrew Morton 3881b00a ef24e0aa

+51 -57
+51 -57
mm/zsmalloc.c
··· 193 193 }; 194 194 }; 195 195 196 + static struct kmem_cache *handle_cachep; 197 + static struct kmem_cache *zspage_cachep; 198 + 196 199 struct zs_pool { 197 200 const char *name; 198 201 199 202 struct size_class *size_class[ZS_SIZE_CLASSES]; 200 - struct kmem_cache *handle_cachep; 201 - struct kmem_cache *zspage_cachep; 202 203 203 204 atomic_long_t pages_allocated; 204 205 ··· 372 371 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {} 373 372 #endif 374 373 375 - static int create_cache(struct zs_pool *pool) 374 + static unsigned long cache_alloc_handle(gfp_t gfp) 376 375 { 377 - char *name; 376 + gfp = gfp & ~(__GFP_HIGHMEM | __GFP_MOVABLE); 378 377 379 - name = kasprintf(GFP_KERNEL, "zs_handle-%s", pool->name); 380 - if (!name) 381 - return -ENOMEM; 382 - pool->handle_cachep = kmem_cache_create(name, ZS_HANDLE_SIZE, 383 - 0, 0, NULL); 384 - kfree(name); 385 - if (!pool->handle_cachep) 386 - return -EINVAL; 387 - 388 - name = kasprintf(GFP_KERNEL, "zspage-%s", pool->name); 389 - if (!name) 390 - return -ENOMEM; 391 - pool->zspage_cachep = kmem_cache_create(name, sizeof(struct zspage), 392 - 0, 0, NULL); 393 - kfree(name); 394 - if (!pool->zspage_cachep) { 395 - kmem_cache_destroy(pool->handle_cachep); 396 - pool->handle_cachep = NULL; 397 - return -EINVAL; 398 - } 399 - 400 - return 0; 378 + return (unsigned long)kmem_cache_alloc(handle_cachep, gfp); 401 379 } 402 380 403 - static void destroy_cache(struct zs_pool *pool) 381 + static void cache_free_handle(unsigned long handle) 404 382 { 405 - kmem_cache_destroy(pool->handle_cachep); 406 - kmem_cache_destroy(pool->zspage_cachep); 383 + kmem_cache_free(handle_cachep, (void *)handle); 407 384 } 408 385 409 - static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp) 386 + static struct zspage *cache_alloc_zspage(gfp_t gfp) 410 387 { 411 - return (unsigned long)kmem_cache_alloc(pool->handle_cachep, 412 - gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE)); 388 + gfp = gfp & ~(__GFP_HIGHMEM | __GFP_MOVABLE); 389 + 390 + return kmem_cache_zalloc(zspage_cachep, gfp); 413 391 } 414 392 415 - static void cache_free_handle(struct zs_pool *pool, unsigned long handle) 393 + static void cache_free_zspage(struct zspage *zspage) 416 394 { 417 - kmem_cache_free(pool->handle_cachep, (void *)handle); 418 - } 419 - 420 - static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags) 421 - { 422 - return kmem_cache_zalloc(pool->zspage_cachep, 423 - flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE)); 424 - } 425 - 426 - static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage) 427 - { 428 - kmem_cache_free(pool->zspage_cachep, zspage); 395 + kmem_cache_free(zspage_cachep, zspage); 429 396 } 430 397 431 398 /* class->lock(which owns the handle) synchronizes races */ ··· 822 853 zpdesc = next; 823 854 } while (zpdesc != NULL); 824 855 825 - cache_free_zspage(pool, zspage); 856 + cache_free_zspage(zspage); 826 857 827 858 class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); 828 859 atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); ··· 935 966 { 936 967 int i; 937 968 struct zpdesc *zpdescs[ZS_MAX_PAGES_PER_ZSPAGE]; 938 - struct zspage *zspage = cache_alloc_zspage(pool, gfp); 969 + struct zspage *zspage = cache_alloc_zspage(gfp); 939 970 940 971 if (!zspage) 941 972 return NULL; ··· 957 988 zpdesc_dec_zone_page_state(zpdescs[i]); 958 989 free_zpdesc(zpdescs[i]); 959 990 } 960 - cache_free_zspage(pool, zspage); 991 + cache_free_zspage(zspage); 961 992 return NULL; 962 993 } 963 994 __zpdesc_set_zsmalloc(zpdesc); ··· 1308 1339 if (unlikely(size > ZS_MAX_ALLOC_SIZE)) 1309 1340 return (unsigned long)ERR_PTR(-ENOSPC); 1310 1341 1311 - handle = cache_alloc_handle(pool, gfp); 1342 + handle = cache_alloc_handle(gfp); 1312 1343 if (!handle) 1313 1344 return (unsigned long)ERR_PTR(-ENOMEM); 1314 1345 ··· 1332 1363 1333 1364 zspage = alloc_zspage(pool, class, gfp, nid); 1334 1365 if (!zspage) { 1335 - cache_free_handle(pool, handle); 1366 + cache_free_handle(handle); 1336 1367 return (unsigned long)ERR_PTR(-ENOMEM); 1337 1368 } 1338 1369 ··· 1412 1443 free_zspage(pool, class, zspage); 1413 1444 1414 1445 spin_unlock(&class->lock); 1415 - cache_free_handle(pool, handle); 1446 + cache_free_handle(handle); 1416 1447 } 1417 1448 EXPORT_SYMBOL_GPL(zs_free); 1418 1449 ··· 2074 2105 if (!pool->name) 2075 2106 goto err; 2076 2107 2077 - if (create_cache(pool)) 2078 - goto err; 2079 - 2080 2108 /* 2081 2109 * Iterate reversely, because, size of size_class that we want to use 2082 2110 * for merging should be larger or equal to current size. ··· 2195 2229 kfree(class); 2196 2230 } 2197 2231 2198 - destroy_cache(pool); 2199 2232 kfree(pool->name); 2200 2233 kfree(pool); 2201 2234 } 2202 2235 EXPORT_SYMBOL_GPL(zs_destroy_pool); 2203 2236 2237 + static void zs_destroy_caches(void) 2238 + { 2239 + kmem_cache_destroy(handle_cachep); 2240 + handle_cachep = NULL; 2241 + kmem_cache_destroy(zspage_cachep); 2242 + zspage_cachep = NULL; 2243 + } 2244 + 2245 + static int __init zs_init_caches(void) 2246 + { 2247 + handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE, 2248 + 0, 0, NULL); 2249 + zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage), 2250 + 0, 0, NULL); 2251 + 2252 + if (!handle_cachep || !zspage_cachep) { 2253 + zs_destroy_caches(); 2254 + return -ENOMEM; 2255 + } 2256 + return 0; 2257 + } 2258 + 2204 2259 static int __init zs_init(void) 2205 2260 { 2206 - int rc __maybe_unused; 2261 + int rc; 2262 + 2263 + rc = zs_init_caches(); 2264 + if (rc) 2265 + return rc; 2207 2266 2208 2267 #ifdef CONFIG_COMPACTION 2209 2268 rc = set_movable_ops(&zsmalloc_mops, PGTY_zsmalloc); 2210 - if (rc) 2269 + if (rc) { 2270 + zs_destroy_caches(); 2211 2271 return rc; 2272 + } 2212 2273 #endif 2213 2274 zs_stat_init(); 2214 2275 return 0; ··· 2247 2254 set_movable_ops(NULL, PGTY_zsmalloc); 2248 2255 #endif 2249 2256 zs_stat_exit(); 2257 + zs_destroy_caches(); 2250 2258 } 2251 2259 2252 2260 module_init(zs_init);