mm: zswap: interact directly with zsmalloc

+52 -146

1 changed file

expand all

zswap.c

+52 -146

mm/zswap.c

··· 25 25 #include <linux/scatterlist.h> 26 26 #include <linux/mempolicy.h> 27 27 #include <linux/mempool.h> 28 - #include <linux/zpool.h> 29 28 #include <crypto/acompress.h> 30 29 #include <linux/zswap.h> 31 30 #include <linux/mm_types.h> ··· 34 35 #include <linux/pagemap.h> 35 36 #include <linux/workqueue.h> 36 37 #include <linux/list_lru.h> 38 + #include <linux/zsmalloc.h> 37 39 38 40 #include "swap.h" 39 41 #include "internal.h" ··· 107 107 module_param_cb(compressor, &zswap_compressor_param_ops, 108 108 &zswap_compressor, 0644); 109 109 110 - /* Compressed storage zpool to use */ 111 - static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 112 - static int zswap_zpool_param_set(const char *, const struct kernel_param *); 113 - static const struct kernel_param_ops zswap_zpool_param_ops = { 114 - .set = zswap_zpool_param_set, 115 - .get = param_get_charp, 116 - .free = param_free_charp, 117 - }; 118 - module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 119 - 120 110 /* The maximum percentage of memory that the compressed pool can occupy */ 121 111 static unsigned int zswap_max_pool_percent = 20; 122 112 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); ··· 151 161 * needs to be verified that it's still valid in the tree. 152 162 */ 153 163 struct zswap_pool { 154 - struct zpool *zpool; 164 + struct zs_pool *zs_pool; 155 165 struct crypto_acomp_ctx __percpu *acomp_ctx; 156 166 struct percpu_ref ref; 157 167 struct list_head list; ··· 183 193 * logic if referenced is unset. See comments in the shrinker 184 194 * section for context. 185 195 * pool - the zswap_pool the entry's data is in 186 - * handle - zpool allocation handle that stores the compressed page data 196 + * handle - zsmalloc allocation handle that stores the compressed page data 187 197 * objcg - the obj_cgroup that the compressed memory is charged to 188 198 * lru - handle to the pool's lru used to evict pages. 189 199 */ ··· 204 214 static LIST_HEAD(zswap_pools); 205 215 /* protects zswap_pools list modification */ 206 216 static DEFINE_SPINLOCK(zswap_pools_lock); 207 - /* pool counter to provide unique names to zpool */ 217 + /* pool counter to provide unique names to zsmalloc */ 208 218 static atomic_t zswap_pools_count = ATOMIC_INIT(0); 209 219 210 220 enum zswap_init_type { ··· 231 241 >> SWAP_ADDRESS_SPACE_SHIFT]; 232 242 } 233 243 234 - #define zswap_pool_debug(msg, p) \ 235 - pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 236 - zpool_get_type((p)->zpool)) 244 + #define zswap_pool_debug(msg, p) \ 245 + pr_debug("%s pool %s\n", msg, (p)->tfm_name) 237 246 238 247 /********************************* 239 248 * pool functions 240 249 **********************************/ 241 250 static void __zswap_pool_empty(struct percpu_ref *ref); 242 251 243 - static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 252 + static struct zswap_pool *zswap_pool_create(char *compressor) 244 253 { 245 254 struct zswap_pool *pool; 246 255 char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 247 - gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 248 256 int ret, cpu; 249 257 250 - if (!zswap_has_pool) { 251 - /* if either are unset, pool initialization failed, and we 252 - * need both params to be set correctly before trying to 253 - * create a pool. 254 - */ 255 - if (!strcmp(type, ZSWAP_PARAM_UNSET)) 256 - return NULL; 257 - if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 258 - return NULL; 259 - } 258 + if (!zswap_has_pool && !strcmp(compressor, ZSWAP_PARAM_UNSET)) 259 + return NULL; 260 260 261 261 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 262 262 if (!pool) ··· 254 274 255 275 /* unique name for each pool specifically required by zsmalloc */ 256 276 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count)); 257 - pool->zpool = zpool_create_pool(type, name, gfp); 258 - if (!pool->zpool) { 259 - pr_err("%s zpool not available\n", type); 277 + pool->zs_pool = zs_create_pool(name); 278 + if (!pool->zs_pool) 260 279 goto error; 261 - } 262 - pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); 263 280 264 281 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 265 282 ··· 292 315 error: 293 316 if (pool->acomp_ctx) 294 317 free_percpu(pool->acomp_ctx); 295 - if (pool->zpool) 296 - zpool_destroy_pool(pool->zpool); 318 + if (pool->zs_pool) 319 + zs_destroy_pool(pool->zs_pool); 297 320 kfree(pool); 298 321 return NULL; 299 322 } 300 323 301 324 static struct zswap_pool *__zswap_pool_create_fallback(void) 302 325 { 303 - bool has_comp, has_zpool; 304 - 305 - has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 306 - if (!has_comp && strcmp(zswap_compressor, 307 - CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 326 + if (!crypto_has_acomp(zswap_compressor, 0, 0) && 327 + strcmp(zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 308 328 pr_err("compressor %s not available, using default %s\n", 309 329 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); 310 330 param_free_charp(&zswap_compressor); 311 331 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 312 - has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 313 332 } 314 - if (!has_comp) { 315 - pr_err("default compressor %s not available\n", 316 - zswap_compressor); 317 - param_free_charp(&zswap_compressor); 333 + 334 + /* Default compressor should be available. Kconfig bug? */ 335 + if (WARN_ON_ONCE(!crypto_has_acomp(zswap_compressor, 0, 0))) { 318 336 zswap_compressor = ZSWAP_PARAM_UNSET; 319 - } 320 - 321 - has_zpool = zpool_has_pool(zswap_zpool_type); 322 - if (!has_zpool && strcmp(zswap_zpool_type, 323 - CONFIG_ZSWAP_ZPOOL_DEFAULT)) { 324 - pr_err("zpool %s not available, using default %s\n", 325 - zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); 326 - param_free_charp(&zswap_zpool_type); 327 - zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 328 - has_zpool = zpool_has_pool(zswap_zpool_type); 329 - } 330 - if (!has_zpool) { 331 - pr_err("default zpool %s not available\n", 332 - zswap_zpool_type); 333 - param_free_charp(&zswap_zpool_type); 334 - zswap_zpool_type = ZSWAP_PARAM_UNSET; 335 - } 336 - 337 - if (!has_comp || !has_zpool) 338 337 return NULL; 338 + } 339 339 340 - return zswap_pool_create(zswap_zpool_type, zswap_compressor); 340 + return zswap_pool_create(zswap_compressor); 341 341 } 342 342 343 343 static void zswap_pool_destroy(struct zswap_pool *pool) ··· 324 370 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 325 371 free_percpu(pool->acomp_ctx); 326 372 327 - zpool_destroy_pool(pool->zpool); 373 + zs_destroy_pool(pool->zs_pool); 328 374 kfree(pool); 329 375 } 330 376 ··· 416 462 } 417 463 418 464 /* type and compressor must be null-terminated */ 419 - static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 465 + static struct zswap_pool *zswap_pool_find_get(char *compressor) 420 466 { 421 467 struct zswap_pool *pool; 422 468 ··· 424 470 425 471 list_for_each_entry_rcu(pool, &zswap_pools, list) { 426 472 if (strcmp(pool->tfm_name, compressor)) 427 - continue; 428 - if (strcmp(zpool_get_type(pool->zpool), type)) 429 473 continue; 430 474 /* if we can't get it, it's about to be destroyed */ 431 475 if (!zswap_pool_tryget(pool)) ··· 451 499 452 500 rcu_read_lock(); 453 501 list_for_each_entry_rcu(pool, &zswap_pools, list) 454 - total += zpool_get_total_pages(pool->zpool); 502 + total += zs_get_total_pages(pool->zs_pool); 455 503 rcu_read_unlock(); 456 504 457 505 return total; ··· 476 524 * param callbacks 477 525 **********************************/ 478 526 479 - static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) 480 - { 481 - /* no change required */ 482 - if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 483 - return false; 484 - return true; 485 - } 486 - 487 - /* val must be a null-terminated string */ 488 - static int __zswap_param_set(const char *val, const struct kernel_param *kp, 489 - char *type, char *compressor) 527 + static int zswap_compressor_param_set(const char *val, const struct kernel_param *kp) 490 528 { 491 529 struct zswap_pool *pool, *put_pool = NULL; 492 530 char *s = strstrip((char *)val); 531 + bool create_pool = false; 493 532 int ret = 0; 494 - bool new_pool = false; 495 533 496 534 mutex_lock(&zswap_init_lock); 497 535 switch (zswap_init_state) { 498 536 case ZSWAP_UNINIT: 499 - /* if this is load-time (pre-init) param setting, 500 - * don't create a pool; that's done during init. 501 - */ 537 + /* Handled in zswap_setup() */ 502 538 ret = param_set_charp(s, kp); 503 539 break; 504 540 case ZSWAP_INIT_SUCCEED: 505 - new_pool = zswap_pool_changed(s, kp); 541 + if (!zswap_has_pool || strcmp(s, *(char **)kp->arg)) 542 + create_pool = true; 506 543 break; 507 544 case ZSWAP_INIT_FAILED: 508 545 pr_err("can't set param, initialization failed\n"); ··· 499 558 } 500 559 mutex_unlock(&zswap_init_lock); 501 560 502 - /* no need to create a new pool, return directly */ 503 - if (!new_pool) 561 + if (!create_pool) 504 562 return ret; 505 563 506 - if (!type) { 507 - if (!zpool_has_pool(s)) { 508 - pr_err("zpool %s not available\n", s); 509 - return -ENOENT; 510 - } 511 - type = s; 512 - } else if (!compressor) { 513 - if (!crypto_has_acomp(s, 0, 0)) { 514 - pr_err("compressor %s not available\n", s); 515 - return -ENOENT; 516 - } 517 - compressor = s; 518 - } else { 519 - WARN_ON(1); 520 - return -EINVAL; 564 + if (!crypto_has_acomp(s, 0, 0)) { 565 + pr_err("compressor %s not available\n", s); 566 + return -ENOENT; 521 567 } 522 568 523 569 spin_lock_bh(&zswap_pools_lock); 524 570 525 - pool = zswap_pool_find_get(type, compressor); 571 + pool = zswap_pool_find_get(s); 526 572 if (pool) { 527 573 zswap_pool_debug("using existing", pool); 528 574 WARN_ON(pool == zswap_pool_current()); ··· 519 591 spin_unlock_bh(&zswap_pools_lock); 520 592 521 593 if (!pool) 522 - pool = zswap_pool_create(type, compressor); 594 + pool = zswap_pool_create(s); 523 595 else { 524 596 /* 525 597 * Restore the initial ref dropped by percpu_ref_kill() ··· 544 616 list_add_rcu(&pool->list, &zswap_pools); 545 617 zswap_has_pool = true; 546 618 } else if (pool) { 547 - /* add the possibly pre-existing pool to the end of the pools 619 + /* 620 + * Add the possibly pre-existing pool to the end of the pools 548 621 * list; if it's new (and empty) then it'll be removed and 549 622 * destroyed by the put after we drop the lock 550 623 */ ··· 555 626 556 627 spin_unlock_bh(&zswap_pools_lock); 557 628 558 - if (!zswap_has_pool && !pool) { 559 - /* if initial pool creation failed, and this pool creation also 560 - * failed, maybe both compressor and zpool params were bad. 561 - * Allow changing this param, so pool creation will succeed 562 - * when the other param is changed. We already verified this 563 - * param is ok in the zpool_has_pool() or crypto_has_acomp() 564 - * checks above. 565 - */ 566 - ret = param_set_charp(s, kp); 567 - } 568 - 569 - /* drop the ref from either the old current pool, 629 + /* 630 + * Drop the ref from either the old current pool, 570 631 * or the new pool we failed to add 571 632 */ 572 633 if (put_pool) 573 634 percpu_ref_kill(&put_pool->ref); 574 635 575 636 return ret; 576 - } 577 - 578 - static int zswap_compressor_param_set(const char *val, 579 - const struct kernel_param *kp) 580 - { 581 - return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 582 - } 583 - 584 - static int zswap_zpool_param_set(const char *val, 585 - const struct kernel_param *kp) 586 - { 587 - return __zswap_param_set(val, kp, NULL, zswap_compressor); 588 637 } 589 638 590 639 static int zswap_enabled_param_set(const char *val, ··· 708 801 } 709 802 710 803 /* 711 - * Carries out the common pattern of freeing and entry's zpool allocation, 804 + * Carries out the common pattern of freeing an entry's zsmalloc allocation, 712 805 * freeing the entry itself, and decrementing the number of stored pages. 713 806 */ 714 807 static void zswap_entry_free(struct zswap_entry *entry) 715 808 { 716 809 zswap_lru_del(&zswap_list_lru, entry); 717 - zpool_free(entry->pool->zpool, entry->handle); 810 + zs_free(entry->pool->zs_pool, entry->handle); 718 811 zswap_pool_put(entry->pool); 719 812 if (entry->objcg) { 720 813 obj_cgroup_uncharge_zswap(entry->objcg, entry->length); ··· 856 949 int comp_ret = 0, alloc_ret = 0; 857 950 unsigned int dlen = PAGE_SIZE; 858 951 unsigned long handle; 859 - struct zpool *zpool; 860 952 gfp_t gfp; 861 953 u8 *dst; 862 954 bool mapped = false; ··· 903 997 mapped = true; 904 998 } 905 999 906 - zpool = pool->zpool; 907 1000 gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE; 908 - alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle, page_to_nid(page)); 909 - if (alloc_ret) 1001 + handle = zs_malloc(pool->zs_pool, dlen, gfp, page_to_nid(page)); 1002 + if (IS_ERR_VALUE(handle)) { 1003 + alloc_ret = PTR_ERR((void *)handle); 910 1004 goto unlock; 1005 + } 911 1006 912 - zpool_obj_write(zpool, handle, dst, dlen); 1007 + zs_obj_write(pool->zs_pool, handle, dst, dlen); 913 1008 entry->handle = handle; 914 1009 entry->length = dlen; 915 1010 ··· 930 1023 931 1024 static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) 932 1025 { 933 - struct zpool *zpool = entry->pool->zpool; 1026 + struct zswap_pool *pool = entry->pool; 934 1027 struct scatterlist input, output; 935 1028 struct crypto_acomp_ctx *acomp_ctx; 936 1029 int decomp_ret = 0, dlen = PAGE_SIZE; 937 1030 u8 *src, *obj; 938 1031 939 - acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool); 940 - obj = zpool_obj_read_begin(zpool, entry->handle, acomp_ctx->buffer); 1032 + acomp_ctx = acomp_ctx_get_cpu_lock(pool); 1033 + obj = zs_obj_read_begin(pool->zs_pool, entry->handle, acomp_ctx->buffer); 941 1034 942 1035 /* zswap entries of length PAGE_SIZE are not compressed. */ 943 1036 if (entry->length == PAGE_SIZE) { ··· 946 1039 } 947 1040 948 1041 /* 949 - * zpool_obj_read_begin() might return a kmap address of highmem when 1042 + * zs_obj_read_begin() might return a kmap address of highmem when 950 1043 * acomp_ctx->buffer is not used. However, sg_init_one() does not 951 1044 * handle highmem addresses, so copy the object to acomp_ctx->buffer. 952 1045 */ ··· 966 1059 dlen = acomp_ctx->req->dlen; 967 1060 968 1061 read_done: 969 - zpool_obj_read_end(zpool, entry->handle, obj); 1062 + zs_obj_read_end(pool->zs_pool, entry->handle, obj); 970 1063 acomp_ctx_put_unlock(acomp_ctx); 971 1064 972 1065 if (!decomp_ret && dlen == PAGE_SIZE) ··· 1483 1576 return true; 1484 1577 1485 1578 store_failed: 1486 - zpool_free(pool->zpool, entry->handle); 1579 + zs_free(pool->zs_pool, entry->handle); 1487 1580 compress_failed: 1488 1581 zswap_entry_cache_free(entry); 1489 1582 return false; ··· 1813 1906 1814 1907 pool = __zswap_pool_create_fallback(); 1815 1908 if (pool) { 1816 - pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1817 - zpool_get_type(pool->zpool)); 1909 + pr_info("loaded using pool %s\n", pool->tfm_name); 1818 1910 list_add(&pool->list, &zswap_pools); 1819 1911 zswap_has_pool = true; 1820 1912 static_branch_enable(&zswap_ever_enabled);

Configure Feed

Configure Feed