tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

zsmalloc: use actual object size to detect spans

Using class->size to detect spanning objects is not entirely correct,
because some size classes can hold a range of object sizes of up to
class->size bytes in length, due to size-classes merge. Such classes use
padding for cases when actually written objects are smaller than
class->size. zs_obj_read_begin() can incorrectly hit the slow path and
perform memcpy of such objects, basically copying padding bytes. Instead
of class->size zs_obj_read_begin() should use the actual compressed object
length (both zram and zswap know it) so that it can correctly handle
situations when a written object is small enough to fit into the first
physical page.

Link: https://lkml.kernel.org/r/20260107052145.3586917-1-senozhatsky@chromium.org
Signed-off-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Reviewed-by: Yosry Ahmed <yosry.ahmed@linux.dev> [zsmalloc & zswap]
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Cc: Brian Geffon <bgeffon@google.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Sergey Senozhatsky and committed by
Andrew Morton 0be909f1 95296536

+24 -15
+8 -6
drivers/block/zram/zram_drv.c
··· 2062 2062 void *src, *dst; 2063 2063 2064 2064 handle = get_slot_handle(zram, index); 2065 - src = zs_obj_read_begin(zram->mem_pool, handle, NULL); 2065 + src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL); 2066 2066 dst = kmap_local_page(page); 2067 2067 copy_page(dst, src); 2068 2068 kunmap_local(dst); 2069 - zs_obj_read_end(zram->mem_pool, handle, src); 2069 + zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src); 2070 2070 2071 2071 return 0; 2072 2072 } ··· 2084 2084 prio = get_slot_comp_priority(zram, index); 2085 2085 2086 2086 zstrm = zcomp_stream_get(zram->comps[prio]); 2087 - src = zs_obj_read_begin(zram->mem_pool, handle, zstrm->local_copy); 2087 + src = zs_obj_read_begin(zram->mem_pool, handle, size, 2088 + zstrm->local_copy); 2088 2089 dst = kmap_local_page(page); 2089 2090 ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst); 2090 2091 kunmap_local(dst); 2091 - zs_obj_read_end(zram->mem_pool, handle, src); 2092 + zs_obj_read_end(zram->mem_pool, handle, size, src); 2092 2093 zcomp_stream_put(zstrm); 2093 2094 2094 2095 return ret; ··· 2112 2111 * takes place here, as we read raw compressed data. 2113 2112 */ 2114 2113 zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]); 2115 - src = zs_obj_read_begin(zram->mem_pool, handle, zstrm->local_copy); 2114 + src = zs_obj_read_begin(zram->mem_pool, handle, size, 2115 + zstrm->local_copy); 2116 2116 memcpy_to_page(page, 0, src, size); 2117 - zs_obj_read_end(zram->mem_pool, handle, src); 2117 + zs_obj_read_end(zram->mem_pool, handle, size, src); 2118 2118 zcomp_stream_put(zstrm); 2119 2119 2120 2120 return 0;
+2 -2
include/linux/zsmalloc.h
··· 40 40 void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats); 41 41 42 42 void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, 43 - void *local_copy); 43 + size_t mem_len, void *local_copy); 44 44 void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, 45 - void *handle_mem); 45 + size_t mem_len, void *handle_mem); 46 46 void zs_obj_write(struct zs_pool *pool, unsigned long handle, 47 47 void *handle_mem, size_t mem_len); 48 48
+11 -5
mm/zsmalloc.c
··· 1065 1065 EXPORT_SYMBOL_GPL(zs_get_total_pages); 1066 1066 1067 1067 void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, 1068 - void *local_copy) 1068 + size_t mem_len, void *local_copy) 1069 1069 { 1070 1070 struct zspage *zspage; 1071 1071 struct zpdesc *zpdesc; ··· 1087 1087 class = zspage_class(pool, zspage); 1088 1088 off = offset_in_page(class->size * obj_idx); 1089 1089 1090 - if (off + class->size <= PAGE_SIZE) { 1090 + if (!ZsHugePage(zspage)) 1091 + mem_len += ZS_HANDLE_SIZE; 1092 + 1093 + if (off + mem_len <= PAGE_SIZE) { 1091 1094 /* this object is contained entirely within a page */ 1092 1095 addr = kmap_local_zpdesc(zpdesc); 1093 1096 addr += off; ··· 1099 1096 1100 1097 /* this object spans two pages */ 1101 1098 sizes[0] = PAGE_SIZE - off; 1102 - sizes[1] = class->size - sizes[0]; 1099 + sizes[1] = mem_len - sizes[0]; 1103 1100 addr = local_copy; 1104 1101 1105 1102 memcpy_from_page(addr, zpdesc_page(zpdesc), ··· 1118 1115 EXPORT_SYMBOL_GPL(zs_obj_read_begin); 1119 1116 1120 1117 void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, 1121 - void *handle_mem) 1118 + size_t mem_len, void *handle_mem) 1122 1119 { 1123 1120 struct zspage *zspage; 1124 1121 struct zpdesc *zpdesc; ··· 1132 1129 class = zspage_class(pool, zspage); 1133 1130 off = offset_in_page(class->size * obj_idx); 1134 1131 1135 - if (off + class->size <= PAGE_SIZE) { 1132 + if (!ZsHugePage(zspage)) 1133 + mem_len += ZS_HANDLE_SIZE; 1134 + 1135 + if (off + mem_len <= PAGE_SIZE) { 1136 1136 if (!ZsHugePage(zspage)) 1137 1137 off += ZS_HANDLE_SIZE; 1138 1138 handle_mem -= off;
+3 -2
mm/zswap.c
··· 937 937 u8 *src, *obj; 938 938 939 939 acomp_ctx = acomp_ctx_get_cpu_lock(pool); 940 - obj = zs_obj_read_begin(pool->zs_pool, entry->handle, acomp_ctx->buffer); 940 + obj = zs_obj_read_begin(pool->zs_pool, entry->handle, entry->length, 941 + acomp_ctx->buffer); 941 942 942 943 /* zswap entries of length PAGE_SIZE are not compressed. */ 943 944 if (entry->length == PAGE_SIZE) { ··· 967 966 dlen = acomp_ctx->req->dlen; 968 967 969 968 read_done: 970 - zs_obj_read_end(pool->zs_pool, entry->handle, obj); 969 + zs_obj_read_end(pool->zs_pool, entry->handle, entry->length, obj); 971 970 acomp_ctx_put_unlock(acomp_ctx); 972 971 973 972 if (!decomp_ret && dlen == PAGE_SIZE)