tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

bcache: remove heap-related macros and switch to generic min_heap

Drop the heap-related macros from bcache and replacing them with the
generic min_heap implementation from include/linux. By doing so, code
readability is improved by using functions instead of macros. Moreover,
the min_heap implementation in include/linux adopts a bottom-up variation
compared to the textbook version currently used in bcache. This bottom-up
variation allows for approximately 50% reduction in the number of
comparison operations during heap siftdown, without changing the number of
swaps, thus making it more efficient.

Link: https://lkml.kernel.org/ioyfizrzq7w7mjrqcadtzsfgpuntowtjdw5pgn4qhvsdp4mqqg@nrlek5vmisbu
Link: https://lkml.kernel.org/r/20240524152958.919343-16-visitorckw@gmail.com
Signed-off-by: Kuan-Wei Chiu <visitorckw@gmail.com>
Reviewed-by: Ian Rogers <irogers@google.com>
Acked-by: Coly Li <colyli@suse.de>
Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Bagas Sanjaya <bagasdotme@gmail.com>
Cc: Brian Foster <bfoster@redhat.com>
Cc: Ching-Chun (Jim) Huang <jserv@ccns.ncku.edu.tw>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Matthew Sakai <msakai@redhat.com>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Randy Dunlap <rdunlap@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Kuan-Wei Chiu and committed by
Andrew Morton 866898ef 7099f74d

+263 -217
+47 -17
drivers/md/bcache/alloc.c
··· 164 164 * prio is worth 1/8th of what INITIAL_PRIO is worth. 165 165 */ 166 166 167 - #define bucket_prio(b) \ 168 - ({ \ 169 - unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; \ 170 - \ 171 - (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); \ 172 - }) 167 + static inline unsigned int new_bucket_prio(struct cache *ca, struct bucket *b) 168 + { 169 + unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; 173 170 174 - #define bucket_max_cmp(l, r) (bucket_prio(l) < bucket_prio(r)) 175 - #define bucket_min_cmp(l, r) (bucket_prio(l) > bucket_prio(r)) 171 + return (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); 172 + } 173 + 174 + static inline bool new_bucket_max_cmp(const void *l, const void *r, void *args) 175 + { 176 + struct bucket **lhs = (struct bucket **)l; 177 + struct bucket **rhs = (struct bucket **)r; 178 + struct cache *ca = args; 179 + 180 + return new_bucket_prio(ca, *lhs) > new_bucket_prio(ca, *rhs); 181 + } 182 + 183 + static inline bool new_bucket_min_cmp(const void *l, const void *r, void *args) 184 + { 185 + struct bucket **lhs = (struct bucket **)l; 186 + struct bucket **rhs = (struct bucket **)r; 187 + struct cache *ca = args; 188 + 189 + return new_bucket_prio(ca, *lhs) < new_bucket_prio(ca, *rhs); 190 + } 191 + 192 + static inline void new_bucket_swap(void *l, void *r, void __always_unused *args) 193 + { 194 + struct bucket **lhs = l, **rhs = r; 195 + 196 + swap(*lhs, *rhs); 197 + } 176 198 177 199 static void invalidate_buckets_lru(struct cache *ca) 178 200 { 179 201 struct bucket *b; 180 - ssize_t i; 202 + const struct min_heap_callbacks bucket_max_cmp_callback = { 203 + .less = new_bucket_max_cmp, 204 + .swp = new_bucket_swap, 205 + }; 206 + const struct min_heap_callbacks bucket_min_cmp_callback = { 207 + .less = new_bucket_min_cmp, 208 + .swp = new_bucket_swap, 209 + }; 181 210 182 - ca->heap.used = 0; 211 + ca->heap.nr = 0; 183 212 184 213 for_each_bucket(b, ca) { 185 214 if (!bch_can_invalidate_bucket(ca, b)) 186 215 continue; 187 216 188 - if (!heap_full(&ca->heap)) 189 - heap_add(&ca->heap, b, bucket_max_cmp); 190 - else if (bucket_max_cmp(b, heap_peek(&ca->heap))) { 217 + if (!min_heap_full(&ca->heap)) 218 + min_heap_push(&ca->heap, &b, &bucket_max_cmp_callback, ca); 219 + else if (!new_bucket_max_cmp(&b, min_heap_peek(&ca->heap), ca)) { 191 220 ca->heap.data[0] = b; 192 - heap_sift(&ca->heap, 0, bucket_max_cmp); 221 + min_heap_sift_down(&ca->heap, 0, &bucket_max_cmp_callback, ca); 193 222 } 194 223 } 195 224 196 - for (i = ca->heap.used / 2 - 1; i >= 0; --i) 197 - heap_sift(&ca->heap, i, bucket_min_cmp); 225 + min_heapify_all(&ca->heap, &bucket_min_cmp_callback, ca); 198 226 199 227 while (!fifo_full(&ca->free_inc)) { 200 - if (!heap_pop(&ca->heap, b, bucket_min_cmp)) { 228 + if (!ca->heap.nr) { 201 229 /* 202 230 * We don't want to be calling invalidate_buckets() 203 231 * multiple times when it can't do anything ··· 234 206 wake_up_gc(ca->set); 235 207 return; 236 208 } 209 + b = min_heap_peek(&ca->heap)[0]; 210 + min_heap_pop(&ca->heap, &bucket_min_cmp_callback, ca); 237 211 238 212 bch_invalidate_one_bucket(ca, b); 239 213 }
+1 -1
drivers/md/bcache/bcache.h
··· 458 458 /* Allocation stuff: */ 459 459 struct bucket *buckets; 460 460 461 - DECLARE_HEAP(struct bucket *, heap); 461 + DEFINE_MIN_HEAP(struct bucket *, cache_heap) heap; 462 462 463 463 /* 464 464 * If nonzero, we know we aren't going to find any buckets to invalidate
+79 -45
drivers/md/bcache/bset.c
··· 54 54 int __bch_count_data(struct btree_keys *b) 55 55 { 56 56 unsigned int ret = 0; 57 - struct btree_iter_stack iter; 57 + struct btree_iter iter; 58 58 struct bkey *k; 59 + 60 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 59 61 60 62 if (b->ops->is_extents) 61 63 for_each_key(b, k, &iter) ··· 69 67 { 70 68 va_list args; 71 69 struct bkey *k, *p = NULL; 72 - struct btree_iter_stack iter; 70 + struct btree_iter iter; 73 71 const char *err; 72 + 73 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 74 74 75 75 for_each_key(b, k, &iter) { 76 76 if (b->ops->is_extents) { ··· 114 110 115 111 static void bch_btree_iter_next_check(struct btree_iter *iter) 116 112 { 117 - struct bkey *k = iter->data->k, *next = bkey_next(k); 113 + struct bkey *k = iter->heap.data->k, *next = bkey_next(k); 118 114 119 - if (next < iter->data->end && 115 + if (next < iter->heap.data->end && 120 116 bkey_cmp(k, iter->b->ops->is_extents ? 121 117 &START_KEY(next) : next) > 0) { 122 118 bch_dump_bucket(iter->b); ··· 883 879 unsigned int status = BTREE_INSERT_STATUS_NO_INSERT; 884 880 struct bset *i = bset_tree_last(b)->data; 885 881 struct bkey *m, *prev = NULL; 886 - struct btree_iter_stack iter; 882 + struct btree_iter iter; 887 883 struct bkey preceding_key_on_stack = ZERO_KEY; 888 884 struct bkey *preceding_key_p = &preceding_key_on_stack; 889 885 890 886 BUG_ON(b->ops->is_extents && !KEY_SIZE(k)); 887 + 888 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 891 889 892 890 /* 893 891 * If k has preceding key, preceding_key_p will be set to address ··· 901 895 else 902 896 preceding_key(k, &preceding_key_p); 903 897 904 - m = bch_btree_iter_stack_init(b, &iter, preceding_key_p); 898 + m = bch_btree_iter_init(b, &iter, preceding_key_p); 905 899 906 - if (b->ops->insert_fixup(b, k, &iter.iter, replace_key)) 900 + if (b->ops->insert_fixup(b, k, &iter, replace_key)) 907 901 return status; 908 902 909 903 status = BTREE_INSERT_STATUS_INSERT; ··· 1083 1077 1084 1078 /* Btree iterator */ 1085 1079 1086 - typedef bool (btree_iter_cmp_fn)(struct btree_iter_set, 1087 - struct btree_iter_set); 1080 + typedef bool (new_btree_iter_cmp_fn)(const void *, const void *, void *); 1088 1081 1089 - static inline bool btree_iter_cmp(struct btree_iter_set l, 1090 - struct btree_iter_set r) 1082 + static inline bool new_btree_iter_cmp(const void *l, const void *r, void __always_unused *args) 1091 1083 { 1092 - return bkey_cmp(l.k, r.k) > 0; 1084 + const struct btree_iter_set *_l = l; 1085 + const struct btree_iter_set *_r = r; 1086 + 1087 + return bkey_cmp(_l->k, _r->k) <= 0; 1088 + } 1089 + 1090 + static inline void new_btree_iter_swap(void *iter1, void *iter2, void __always_unused *args) 1091 + { 1092 + struct btree_iter_set *_iter1 = iter1; 1093 + struct btree_iter_set *_iter2 = iter2; 1094 + 1095 + swap(*_iter1, *_iter2); 1093 1096 } 1094 1097 1095 1098 static inline bool btree_iter_end(struct btree_iter *iter) 1096 1099 { 1097 - return !iter->used; 1100 + return !iter->heap.nr; 1098 1101 } 1099 1102 1100 1103 void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k, 1101 1104 struct bkey *end) 1102 1105 { 1106 + const struct min_heap_callbacks callbacks = { 1107 + .less = new_btree_iter_cmp, 1108 + .swp = new_btree_iter_swap, 1109 + }; 1110 + 1103 1111 if (k != end) 1104 - BUG_ON(!heap_add(iter, 1105 - ((struct btree_iter_set) { k, end }), 1106 - btree_iter_cmp)); 1112 + BUG_ON(!min_heap_push(&iter->heap, 1113 + &((struct btree_iter_set) { k, end }), 1114 + &callbacks, 1115 + NULL)); 1107 1116 } 1108 1117 1109 - static struct bkey *__bch_btree_iter_stack_init(struct btree_keys *b, 1110 - struct btree_iter_stack *iter, 1111 - struct bkey *search, 1112 - struct bset_tree *start) 1118 + static struct bkey *__bch_btree_iter_init(struct btree_keys *b, 1119 + struct btree_iter *iter, 1120 + struct bkey *search, 1121 + struct bset_tree *start) 1113 1122 { 1114 1123 struct bkey *ret = NULL; 1115 1124 1116 - iter->iter.size = ARRAY_SIZE(iter->stack_data); 1117 - iter->iter.used = 0; 1125 + iter->heap.size = ARRAY_SIZE(iter->heap.preallocated); 1126 + iter->heap.nr = 0; 1118 1127 1119 1128 #ifdef CONFIG_BCACHE_DEBUG 1120 - iter->iter.b = b; 1129 + iter->b = b; 1121 1130 #endif 1122 1131 1123 1132 for (; start <= bset_tree_last(b); start++) { 1124 1133 ret = bch_bset_search(b, start, search); 1125 - bch_btree_iter_push(&iter->iter, ret, bset_bkey_last(start->data)); 1134 + bch_btree_iter_push(iter, ret, bset_bkey_last(start->data)); 1126 1135 } 1127 1136 1128 1137 return ret; 1129 1138 } 1130 1139 1131 - struct bkey *bch_btree_iter_stack_init(struct btree_keys *b, 1132 - struct btree_iter_stack *iter, 1140 + struct bkey *bch_btree_iter_init(struct btree_keys *b, 1141 + struct btree_iter *iter, 1133 1142 struct bkey *search) 1134 1143 { 1135 - return __bch_btree_iter_stack_init(b, iter, search, b->set); 1144 + return __bch_btree_iter_init(b, iter, search, b->set); 1136 1145 } 1137 1146 1138 1147 static inline struct bkey *__bch_btree_iter_next(struct btree_iter *iter, 1139 - btree_iter_cmp_fn *cmp) 1148 + new_btree_iter_cmp_fn *cmp) 1140 1149 { 1141 1150 struct btree_iter_set b __maybe_unused; 1142 1151 struct bkey *ret = NULL; 1152 + const struct min_heap_callbacks callbacks = { 1153 + .less = cmp, 1154 + .swp = new_btree_iter_swap, 1155 + }; 1143 1156 1144 1157 if (!btree_iter_end(iter)) { 1145 1158 bch_btree_iter_next_check(iter); 1146 1159 1147 - ret = iter->data->k; 1148 - iter->data->k = bkey_next(iter->data->k); 1160 + ret = iter->heap.data->k; 1161 + iter->heap.data->k = bkey_next(iter->heap.data->k); 1149 1162 1150 - if (iter->data->k > iter->data->end) { 1163 + if (iter->heap.data->k > iter->heap.data->end) { 1151 1164 WARN_ONCE(1, "bset was corrupt!\n"); 1152 - iter->data->k = iter->data->end; 1165 + iter->heap.data->k = iter->heap.data->end; 1153 1166 } 1154 1167 1155 - if (iter->data->k == iter->data->end) 1156 - heap_pop(iter, b, cmp); 1168 + if (iter->heap.data->k == iter->heap.data->end) { 1169 + if (iter->heap.nr) { 1170 + b = min_heap_peek(&iter->heap)[0]; 1171 + min_heap_pop(&iter->heap, &callbacks, NULL); 1172 + } 1173 + } 1157 1174 else 1158 - heap_sift(iter, 0, cmp); 1175 + min_heap_sift_down(&iter->heap, 0, &callbacks, NULL); 1159 1176 } 1160 1177 1161 1178 return ret; ··· 1186 1157 1187 1158 struct bkey *bch_btree_iter_next(struct btree_iter *iter) 1188 1159 { 1189 - return __bch_btree_iter_next(iter, btree_iter_cmp); 1160 + return __bch_btree_iter_next(iter, new_btree_iter_cmp); 1190 1161 1191 1162 } 1192 1163 ··· 1224 1195 struct btree_iter *iter, 1225 1196 bool fixup, bool remove_stale) 1226 1197 { 1227 - int i; 1228 1198 struct bkey *k, *last = NULL; 1229 1199 BKEY_PADDED(k) tmp; 1230 1200 bool (*bad)(struct btree_keys *, const struct bkey *) = remove_stale 1231 1201 ? bch_ptr_bad 1232 1202 : bch_ptr_invalid; 1203 + const struct min_heap_callbacks callbacks = { 1204 + .less = b->ops->sort_cmp, 1205 + .swp = new_btree_iter_swap, 1206 + }; 1233 1207 1234 1208 /* Heapify the iterator, using our comparison function */ 1235 - for (i = iter->used / 2 - 1; i >= 0; --i) 1236 - heap_sift(iter, i, b->ops->sort_cmp); 1209 + min_heapify_all(&iter->heap, &callbacks, NULL); 1237 1210 1238 1211 while (!btree_iter_end(iter)) { 1239 1212 if (b->ops->sort_fixup && fixup) ··· 1324 1293 struct bset_sort_state *state) 1325 1294 { 1326 1295 size_t order = b->page_order, keys = 0; 1327 - struct btree_iter_stack iter; 1296 + struct btree_iter iter; 1328 1297 int oldsize = bch_count_data(b); 1329 1298 1330 - __bch_btree_iter_stack_init(b, &iter, NULL, &b->set[start]); 1299 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1300 + __bch_btree_iter_init(b, &iter, NULL, &b->set[start]); 1331 1301 1332 1302 if (start) { 1333 1303 unsigned int i; ··· 1339 1307 order = get_order(__set_bytes(b->set->data, keys)); 1340 1308 } 1341 1309 1342 - __btree_sort(b, &iter.iter, start, order, false, state); 1310 + __btree_sort(b, &iter, start, order, false, state); 1343 1311 1344 1312 EBUG_ON(oldsize >= 0 && bch_count_data(b) != oldsize); 1345 1313 } ··· 1355 1323 struct bset_sort_state *state) 1356 1324 { 1357 1325 uint64_t start_time = local_clock(); 1358 - struct btree_iter_stack iter; 1326 + struct btree_iter iter; 1359 1327 1360 - bch_btree_iter_stack_init(b, &iter, NULL); 1328 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1361 1329 1362 - btree_mergesort(b, new->set->data, &iter.iter, false, true); 1330 + bch_btree_iter_init(b, &iter, NULL); 1331 + 1332 + btree_mergesort(b, new->set->data, &iter, false, true); 1363 1333 1364 1334 bch_time_stats_update(&state->time, start_time); 1365 1335
+17 -23
drivers/md/bcache/bset.h
··· 187 187 }; 188 188 189 189 struct btree_keys_ops { 190 - bool (*sort_cmp)(struct btree_iter_set l, 191 - struct btree_iter_set r); 190 + bool (*sort_cmp)(const void *l, 191 + const void *r, 192 + void *args); 192 193 struct bkey *(*sort_fixup)(struct btree_iter *iter, 193 194 struct bkey *tmp); 194 195 bool (*insert_fixup)(struct btree_keys *b, ··· 313 312 BTREE_INSERT_STATUS_FRONT_MERGE, 314 313 }; 315 314 315 + struct btree_iter_set { 316 + struct bkey *k, *end; 317 + }; 318 + 316 319 /* Btree key iteration */ 317 320 318 321 struct btree_iter { 319 - size_t size, used; 320 322 #ifdef CONFIG_BCACHE_DEBUG 321 323 struct btree_keys *b; 322 324 #endif 323 - struct btree_iter_set { 324 - struct bkey *k, *end; 325 - } data[]; 326 - }; 327 - 328 - /* Fixed-size btree_iter that can be allocated on the stack */ 329 - 330 - struct btree_iter_stack { 331 - struct btree_iter iter; 332 - struct btree_iter_set stack_data[MAX_BSETS]; 325 + MIN_HEAP_PREALLOCATED(struct btree_iter_set, btree_iter_heap, MAX_BSETS) heap; 333 326 }; 334 327 335 328 typedef bool (*ptr_filter_fn)(struct btree_keys *b, const struct bkey *k); ··· 335 340 336 341 void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k, 337 342 struct bkey *end); 338 - struct bkey *bch_btree_iter_stack_init(struct btree_keys *b, 339 - struct btree_iter_stack *iter, 340 - struct bkey *search); 343 + struct bkey *bch_btree_iter_init(struct btree_keys *b, 344 + struct btree_iter *iter, 345 + struct bkey *search); 341 346 342 347 struct bkey *__bch_bset_search(struct btree_keys *b, struct bset_tree *t, 343 348 const struct bkey *search); ··· 352 357 return search ? __bch_bset_search(b, t, search) : t->data->start; 353 358 } 354 359 355 - #define for_each_key_filter(b, k, stack_iter, filter) \ 356 - for (bch_btree_iter_stack_init((b), (stack_iter), NULL); \ 357 - ((k) = bch_btree_iter_next_filter(&((stack_iter)->iter), (b), \ 358 - filter));) 360 + #define for_each_key_filter(b, k, iter, filter) \ 361 + for (bch_btree_iter_init((b), (iter), NULL); \ 362 + ((k) = bch_btree_iter_next_filter((iter), (b), filter));) 359 363 360 - #define for_each_key(b, k, stack_iter) \ 361 - for (bch_btree_iter_stack_init((b), (stack_iter), NULL); \ 362 - ((k) = bch_btree_iter_next(&((stack_iter)->iter)));) 364 + #define for_each_key(b, k, iter) \ 365 + for (bch_btree_iter_init((b), (iter), NULL); \ 366 + ((k) = bch_btree_iter_next(iter));) 363 367 364 368 /* Sorting */ 365 369
+40 -29
drivers/md/bcache/btree.c
··· 149 149 { 150 150 const char *err = "bad btree header"; 151 151 struct bset *i = btree_bset_first(b); 152 - struct btree_iter *iter; 152 + struct btree_iter iter; 153 153 154 154 /* 155 155 * c->fill_iter can allocate an iterator with more memory space 156 156 * than static MAX_BSETS. 157 157 * See the comment arount cache_set->fill_iter. 158 158 */ 159 - iter = mempool_alloc(&b->c->fill_iter, GFP_NOIO); 160 - iter->size = b->c->cache->sb.bucket_size / b->c->cache->sb.block_size; 161 - iter->used = 0; 159 + iter.heap.data = mempool_alloc(&b->c->fill_iter, GFP_NOIO); 160 + iter.heap.size = b->c->cache->sb.bucket_size / b->c->cache->sb.block_size; 161 + iter.heap.nr = 0; 162 162 163 163 #ifdef CONFIG_BCACHE_DEBUG 164 - iter->b = &b->keys; 164 + iter.b = &b->keys; 165 165 #endif 166 166 167 167 if (!i->seq) ··· 199 199 if (i != b->keys.set[0].data && !i->keys) 200 200 goto err; 201 201 202 - bch_btree_iter_push(iter, i->start, bset_bkey_last(i)); 202 + bch_btree_iter_push(&iter, i->start, bset_bkey_last(i)); 203 203 204 204 b->written += set_blocks(i, block_bytes(b->c->cache)); 205 205 } ··· 211 211 if (i->seq == b->keys.set[0].data->seq) 212 212 goto err; 213 213 214 - bch_btree_sort_and_fix_extents(&b->keys, iter, &b->c->sort); 214 + bch_btree_sort_and_fix_extents(&b->keys, &iter, &b->c->sort); 215 215 216 216 i = b->keys.set[0].data; 217 217 err = "short btree key"; ··· 223 223 bch_bset_init_next(&b->keys, write_block(b), 224 224 bset_magic(&b->c->cache->sb)); 225 225 out: 226 - mempool_free(iter, &b->c->fill_iter); 226 + mempool_free(iter.heap.data, &b->c->fill_iter); 227 227 return; 228 228 err: 229 229 set_btree_node_io_error(b); ··· 1309 1309 uint8_t stale = 0; 1310 1310 unsigned int keys = 0, good_keys = 0; 1311 1311 struct bkey *k; 1312 - struct btree_iter_stack iter; 1312 + struct btree_iter iter; 1313 1313 struct bset_tree *t; 1314 + 1315 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1314 1316 1315 1317 gc->nodes++; 1316 1318 ··· 1572 1570 static unsigned int btree_gc_count_keys(struct btree *b) 1573 1571 { 1574 1572 struct bkey *k; 1575 - struct btree_iter_stack iter; 1573 + struct btree_iter iter; 1576 1574 unsigned int ret = 0; 1575 + 1576 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1577 1577 1578 1578 for_each_key_filter(&b->keys, k, &iter, bch_ptr_bad) 1579 1579 ret += bkey_u64s(k); ··· 1615 1611 int ret = 0; 1616 1612 bool should_rewrite; 1617 1613 struct bkey *k; 1618 - struct btree_iter_stack iter; 1614 + struct btree_iter iter; 1619 1615 struct gc_merge_info r[GC_MERGE_NODES]; 1620 1616 struct gc_merge_info *i, *last = r + ARRAY_SIZE(r) - 1; 1621 1617 1622 - bch_btree_iter_stack_init(&b->keys, &iter, &b->c->gc_done); 1618 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1619 + bch_btree_iter_init(&b->keys, &iter, &b->c->gc_done); 1623 1620 1624 1621 for (i = r; i < r + ARRAY_SIZE(r); i++) 1625 1622 i->b = ERR_PTR(-EINTR); 1626 1623 1627 1624 while (1) { 1628 - k = bch_btree_iter_next_filter(&iter.iter, &b->keys, 1629 - bch_ptr_bad); 1625 + k = bch_btree_iter_next_filter(&iter, &b->keys, bch_ptr_bad); 1630 1626 if (k) { 1631 1627 r->b = bch_btree_node_get(b->c, op, k, b->level - 1, 1632 1628 true, b); ··· 1921 1917 { 1922 1918 int ret = 0; 1923 1919 struct bkey *k, *p = NULL; 1924 - struct btree_iter_stack iter; 1920 + struct btree_iter iter; 1921 + 1922 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1925 1923 1926 1924 for_each_key_filter(&b->keys, k, &iter, bch_ptr_invalid) 1927 1925 bch_initial_mark_key(b->c, b->level, k); ··· 1931 1925 bch_initial_mark_key(b->c, b->level + 1, &b->key); 1932 1926 1933 1927 if (b->level) { 1934 - bch_btree_iter_stack_init(&b->keys, &iter, NULL); 1928 + bch_btree_iter_init(&b->keys, &iter, NULL); 1935 1929 1936 1930 do { 1937 - k = bch_btree_iter_next_filter(&iter.iter, &b->keys, 1931 + k = bch_btree_iter_next_filter(&iter, &b->keys, 1938 1932 bch_ptr_bad); 1939 1933 if (k) { 1940 1934 btree_node_prefetch(b, k); ··· 1962 1956 struct btree_check_info *info = arg; 1963 1957 struct btree_check_state *check_state = info->state; 1964 1958 struct cache_set *c = check_state->c; 1965 - struct btree_iter_stack iter; 1959 + struct btree_iter iter; 1966 1960 struct bkey *k, *p; 1967 1961 int cur_idx, prev_idx, skip_nr; 1968 1962 ··· 1970 1964 cur_idx = prev_idx = 0; 1971 1965 ret = 0; 1972 1966 1967 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 1968 + 1973 1969 /* root node keys are checked before thread created */ 1974 - bch_btree_iter_stack_init(&c->root->keys, &iter, NULL); 1975 - k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); 1970 + bch_btree_iter_init(&c->root->keys, &iter, NULL); 1971 + k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad); 1976 1972 BUG_ON(!k); 1977 1973 1978 1974 p = k; ··· 1992 1984 skip_nr = cur_idx - prev_idx; 1993 1985 1994 1986 while (skip_nr) { 1995 - k = bch_btree_iter_next_filter(&iter.iter, 1987 + k = bch_btree_iter_next_filter(&iter, 1996 1988 &c->root->keys, 1997 1989 bch_ptr_bad); 1998 1990 if (k) ··· 2065 2057 int ret = 0; 2066 2058 int i; 2067 2059 struct bkey *k = NULL; 2068 - struct btree_iter_stack iter; 2060 + struct btree_iter iter; 2069 2061 struct btree_check_state check_state; 2062 + 2063 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 2070 2064 2071 2065 /* check and mark root node keys */ 2072 2066 for_each_key_filter(&c->root->keys, k, &iter, bch_ptr_invalid) ··· 2563 2553 2564 2554 if (b->level) { 2565 2555 struct bkey *k; 2566 - struct btree_iter_stack iter; 2556 + struct btree_iter iter; 2567 2557 2568 - bch_btree_iter_stack_init(&b->keys, &iter, from); 2558 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 2559 + bch_btree_iter_init(&b->keys, &iter, from); 2569 2560 2570 - while ((k = bch_btree_iter_next_filter(&iter.iter, &b->keys, 2561 + while ((k = bch_btree_iter_next_filter(&iter, &b->keys, 2571 2562 bch_ptr_bad))) { 2572 2563 ret = bcache_btree(map_nodes_recurse, k, b, 2573 2564 op, from, fn, flags); ··· 2597 2586 { 2598 2587 int ret = MAP_CONTINUE; 2599 2588 struct bkey *k; 2600 - struct btree_iter_stack iter; 2589 + struct btree_iter iter; 2601 2590 2602 - bch_btree_iter_stack_init(&b->keys, &iter, from); 2591 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 2592 + bch_btree_iter_init(&b->keys, &iter, from); 2603 2593 2604 - while ((k = bch_btree_iter_next_filter(&iter.iter, &b->keys, 2605 - bch_ptr_bad))) { 2594 + while ((k = bch_btree_iter_next_filter(&iter, &b->keys, bch_ptr_bad))) { 2606 2595 ret = !b->level 2607 2596 ? fn(op, b, k) 2608 2597 : bcache_btree(map_keys_recurse, k,
+34 -19
drivers/md/bcache/extents.c
··· 33 33 i->k = bkey_next(i->k); 34 34 35 35 if (i->k == i->end) 36 - *i = iter->data[--iter->used]; 36 + *i = iter->heap.data[--iter->heap.nr]; 37 37 } 38 38 39 - static bool bch_key_sort_cmp(struct btree_iter_set l, 40 - struct btree_iter_set r) 39 + static bool new_bch_key_sort_cmp(const void *l, const void *r, void *args) 41 40 { 42 - int64_t c = bkey_cmp(l.k, r.k); 41 + struct btree_iter_set *_l = (struct btree_iter_set *)l; 42 + struct btree_iter_set *_r = (struct btree_iter_set *)r; 43 + int64_t c = bkey_cmp(_l->k, _r->k); 43 44 44 - return c ? c > 0 : l.k < r.k; 45 + return !(c ? c > 0 : _l->k < _r->k); 45 46 } 46 47 47 48 static bool __ptr_invalid(struct cache_set *c, const struct bkey *k) ··· 239 238 } 240 239 241 240 const struct btree_keys_ops bch_btree_keys_ops = { 242 - .sort_cmp = bch_key_sort_cmp, 241 + .sort_cmp = new_bch_key_sort_cmp, 243 242 .insert_fixup = bch_btree_ptr_insert_fixup, 244 243 .key_invalid = bch_btree_ptr_invalid, 245 244 .key_bad = bch_btree_ptr_bad, ··· 256 255 * Necessary for btree_sort_fixup() - if there are multiple keys that compare 257 256 * equal in different sets, we have to process them newest to oldest. 258 257 */ 259 - static bool bch_extent_sort_cmp(struct btree_iter_set l, 260 - struct btree_iter_set r) 261 - { 262 - int64_t c = bkey_cmp(&START_KEY(l.k), &START_KEY(r.k)); 263 258 264 - return c ? c > 0 : l.k < r.k; 259 + static bool new_bch_extent_sort_cmp(const void *l, const void *r, void __always_unused *args) 260 + { 261 + struct btree_iter_set *_l = (struct btree_iter_set *)l; 262 + struct btree_iter_set *_r = (struct btree_iter_set *)r; 263 + int64_t c = bkey_cmp(&START_KEY(_l->k), &START_KEY(_r->k)); 264 + 265 + return !(c ? c > 0 : _l->k < _r->k); 266 + } 267 + 268 + static inline void new_btree_iter_swap(void *iter1, void *iter2, void __always_unused *args) 269 + { 270 + struct btree_iter_set *_iter1 = iter1; 271 + struct btree_iter_set *_iter2 = iter2; 272 + 273 + swap(*_iter1, *_iter2); 265 274 } 266 275 267 276 static struct bkey *bch_extent_sort_fixup(struct btree_iter *iter, 268 277 struct bkey *tmp) 269 278 { 270 - while (iter->used > 1) { 271 - struct btree_iter_set *top = iter->data, *i = top + 1; 279 + const struct min_heap_callbacks callbacks = { 280 + .less = new_bch_extent_sort_cmp, 281 + .swp = new_btree_iter_swap, 282 + }; 283 + while (iter->heap.nr > 1) { 284 + struct btree_iter_set *top = iter->heap.data, *i = top + 1; 272 285 273 - if (iter->used > 2 && 274 - bch_extent_sort_cmp(i[0], i[1])) 286 + if (iter->heap.nr > 2 && 287 + !new_bch_extent_sort_cmp(&i[0], &i[1], NULL)) 275 288 i++; 276 289 277 290 if (bkey_cmp(top->k, &START_KEY(i->k)) <= 0) ··· 293 278 294 279 if (!KEY_SIZE(i->k)) { 295 280 sort_key_next(iter, i); 296 - heap_sift(iter, i - top, bch_extent_sort_cmp); 281 + min_heap_sift_down(&iter->heap, i - top, &callbacks, NULL); 297 282 continue; 298 283 } 299 284 ··· 303 288 else 304 289 bch_cut_front(top->k, i->k); 305 290 306 - heap_sift(iter, i - top, bch_extent_sort_cmp); 291 + min_heap_sift_down(&iter->heap, i - top, &callbacks, NULL); 307 292 } else { 308 293 /* can't happen because of comparison func */ 309 294 BUG_ON(!bkey_cmp(&START_KEY(top->k), &START_KEY(i->k))); ··· 313 298 314 299 bch_cut_back(&START_KEY(i->k), tmp); 315 300 bch_cut_front(i->k, top->k); 316 - heap_sift(iter, 0, bch_extent_sort_cmp); 301 + min_heap_sift_down(&iter->heap, 0, &callbacks, NULL); 317 302 318 303 return tmp; 319 304 } else { ··· 633 618 } 634 619 635 620 const struct btree_keys_ops bch_extent_keys_ops = { 636 - .sort_cmp = bch_extent_sort_cmp, 621 + .sort_cmp = new_bch_extent_sort_cmp, 637 622 .sort_fixup = bch_extent_sort_fixup, 638 623 .insert_fixup = bch_extent_insert_fixup, 639 624 .key_invalid = bch_extent_invalid,
+31 -10
drivers/md/bcache/movinggc.c
··· 182 182 closure_sync(&cl); 183 183 } 184 184 185 - static bool bucket_cmp(struct bucket *l, struct bucket *r) 185 + static bool new_bucket_cmp(const void *l, const void *r, void __always_unused *args) 186 186 { 187 - return GC_SECTORS_USED(l) < GC_SECTORS_USED(r); 187 + struct bucket **_l = (struct bucket **)l; 188 + struct bucket **_r = (struct bucket **)r; 189 + 190 + return GC_SECTORS_USED(*_l) >= GC_SECTORS_USED(*_r); 191 + } 192 + 193 + static void new_bucket_swap(void *l, void *r, void __always_unused *args) 194 + { 195 + struct bucket **_l = l; 196 + struct bucket **_r = r; 197 + 198 + swap(*_l, *_r); 188 199 } 189 200 190 201 static unsigned int bucket_heap_top(struct cache *ca) 191 202 { 192 203 struct bucket *b; 193 204 194 - return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0; 205 + return (b = min_heap_peek(&ca->heap)[0]) ? GC_SECTORS_USED(b) : 0; 195 206 } 196 207 197 208 void bch_moving_gc(struct cache_set *c) ··· 210 199 struct cache *ca = c->cache; 211 200 struct bucket *b; 212 201 unsigned long sectors_to_move, reserve_sectors; 202 + const struct min_heap_callbacks callbacks = { 203 + .less = new_bucket_cmp, 204 + .swp = new_bucket_swap, 205 + }; 213 206 214 207 if (!c->copy_gc_enabled) 215 208 return; ··· 224 209 reserve_sectors = ca->sb.bucket_size * 225 210 fifo_used(&ca->free[RESERVE_MOVINGGC]); 226 211 227 - ca->heap.used = 0; 212 + ca->heap.nr = 0; 228 213 229 214 for_each_bucket(b, ca) { 230 215 if (GC_MARK(b) == GC_MARK_METADATA || ··· 233 218 atomic_read(&b->pin)) 234 219 continue; 235 220 236 - if (!heap_full(&ca->heap)) { 221 + if (!min_heap_full(&ca->heap)) { 237 222 sectors_to_move += GC_SECTORS_USED(b); 238 - heap_add(&ca->heap, b, bucket_cmp); 239 - } else if (bucket_cmp(b, heap_peek(&ca->heap))) { 223 + min_heap_push(&ca->heap, &b, &callbacks, NULL); 224 + } else if (!new_bucket_cmp(&b, min_heap_peek(&ca->heap), ca)) { 240 225 sectors_to_move -= bucket_heap_top(ca); 241 226 sectors_to_move += GC_SECTORS_USED(b); 242 227 243 228 ca->heap.data[0] = b; 244 - heap_sift(&ca->heap, 0, bucket_cmp); 229 + min_heap_sift_down(&ca->heap, 0, &callbacks, NULL); 245 230 } 246 231 } 247 232 248 233 while (sectors_to_move > reserve_sectors) { 249 - heap_pop(&ca->heap, b, bucket_cmp); 234 + if (ca->heap.nr) { 235 + b = min_heap_peek(&ca->heap)[0]; 236 + min_heap_pop(&ca->heap, &callbacks, NULL); 237 + } 250 238 sectors_to_move -= GC_SECTORS_USED(b); 251 239 } 252 240 253 - while (heap_pop(&ca->heap, b, bucket_cmp)) 241 + while (ca->heap.nr) { 242 + b = min_heap_peek(&ca->heap)[0]; 243 + min_heap_pop(&ca->heap, &callbacks, NULL); 254 244 SET_GC_MOVE(b, 1); 245 + } 255 246 256 247 mutex_unlock(&c->bucket_lock); 257 248
+1 -2
drivers/md/bcache/super.c
··· 1914 1914 INIT_LIST_HEAD(&c->btree_cache_freed); 1915 1915 INIT_LIST_HEAD(&c->data_buckets); 1916 1916 1917 - iter_size = sizeof(struct btree_iter) + 1918 - ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) * 1917 + iter_size = ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) * 1919 1918 sizeof(struct btree_iter_set); 1920 1919 1921 1920 c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL);
+3 -1
drivers/md/bcache/sysfs.c
··· 660 660 unsigned int bytes = 0; 661 661 struct bkey *k; 662 662 struct btree *b; 663 - struct btree_iter_stack iter; 663 + struct btree_iter iter; 664 + 665 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 664 666 665 667 goto lock_root; 666 668
+2 -65
drivers/md/bcache/util.h
··· 9 9 #include <linux/kernel.h> 10 10 #include <linux/sched/clock.h> 11 11 #include <linux/llist.h> 12 + #include <linux/min_heap.h> 12 13 #include <linux/ratelimit.h> 13 14 #include <linux/vmalloc.h> 14 15 #include <linux/workqueue.h> ··· 31 30 32 31 #endif 33 32 34 - #define DECLARE_HEAP(type, name) \ 35 - struct { \ 36 - size_t size, used; \ 37 - type *data; \ 38 - } name 39 - 40 33 #define init_heap(heap, _size, gfp) \ 41 34 ({ \ 42 35 size_t _bytes; \ 43 - (heap)->used = 0; \ 36 + (heap)->nr = 0; \ 44 37 (heap)->size = (_size); \ 45 38 _bytes = (heap)->size * sizeof(*(heap)->data); \ 46 39 (heap)->data = kvmalloc(_bytes, (gfp) & GFP_KERNEL); \ ··· 46 51 kvfree((heap)->data); \ 47 52 (heap)->data = NULL; \ 48 53 } while (0) 49 - 50 - #define heap_swap(h, i, j) swap((h)->data[i], (h)->data[j]) 51 - 52 - #define heap_sift(h, i, cmp) \ 53 - do { \ 54 - size_t _r, _j = i; \ 55 - \ 56 - for (; _j * 2 + 1 < (h)->used; _j = _r) { \ 57 - _r = _j * 2 + 1; \ 58 - if (_r + 1 < (h)->used && \ 59 - cmp((h)->data[_r], (h)->data[_r + 1])) \ 60 - _r++; \ 61 - \ 62 - if (cmp((h)->data[_r], (h)->data[_j])) \ 63 - break; \ 64 - heap_swap(h, _r, _j); \ 65 - } \ 66 - } while (0) 67 - 68 - #define heap_sift_down(h, i, cmp) \ 69 - do { \ 70 - while (i) { \ 71 - size_t p = (i - 1) / 2; \ 72 - if (cmp((h)->data[i], (h)->data[p])) \ 73 - break; \ 74 - heap_swap(h, i, p); \ 75 - i = p; \ 76 - } \ 77 - } while (0) 78 - 79 - #define heap_add(h, d, cmp) \ 80 - ({ \ 81 - bool _r = !heap_full(h); \ 82 - if (_r) { \ 83 - size_t _i = (h)->used++; \ 84 - (h)->data[_i] = d; \ 85 - \ 86 - heap_sift_down(h, _i, cmp); \ 87 - heap_sift(h, _i, cmp); \ 88 - } \ 89 - _r; \ 90 - }) 91 - 92 - #define heap_pop(h, d, cmp) \ 93 - ({ \ 94 - bool _r = (h)->used; \ 95 - if (_r) { \ 96 - (d) = (h)->data[0]; \ 97 - (h)->used--; \ 98 - heap_swap(h, 0, (h)->used); \ 99 - heap_sift(h, 0, cmp); \ 100 - } \ 101 - _r; \ 102 - }) 103 - 104 - #define heap_peek(h) ((h)->used ? (h)->data[0] : NULL) 105 - 106 - #define heap_full(h) ((h)->used == (h)->size) 107 54 108 55 #define DECLARE_FIFO(type, name) \ 109 56 struct { \
+8 -5
drivers/md/bcache/writeback.c
··· 908 908 struct dirty_init_thrd_info *info = arg; 909 909 struct bch_dirty_init_state *state = info->state; 910 910 struct cache_set *c = state->c; 911 - struct btree_iter_stack iter; 911 + struct btree_iter iter; 912 912 struct bkey *k, *p; 913 913 int cur_idx, prev_idx, skip_nr; 914 914 915 915 k = p = NULL; 916 916 prev_idx = 0; 917 917 918 - bch_btree_iter_stack_init(&c->root->keys, &iter, NULL); 919 - k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); 918 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 919 + bch_btree_iter_init(&c->root->keys, &iter, NULL); 920 + k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad); 920 921 BUG_ON(!k); 921 922 922 923 p = k; ··· 931 930 skip_nr = cur_idx - prev_idx; 932 931 933 932 while (skip_nr) { 934 - k = bch_btree_iter_next_filter(&iter.iter, 933 + k = bch_btree_iter_next_filter(&iter, 935 934 &c->root->keys, 936 935 bch_ptr_bad); 937 936 if (k) ··· 980 979 int i; 981 980 struct btree *b = NULL; 982 981 struct bkey *k = NULL; 983 - struct btree_iter_stack iter; 982 + struct btree_iter iter; 984 983 struct sectors_dirty_init op; 985 984 struct cache_set *c = d->c; 986 985 struct bch_dirty_init_state state; 986 + 987 + min_heap_init(&iter.heap, NULL, MAX_BSETS); 987 988 988 989 retry_lock: 989 990 b = c->root;