Merge tag 'bcachefs-2025-04-03' of git://evilpiepirate.org/bcachefs

+1

fs/bcachefs/Kconfig

··· 26 26 select SRCU 27 27 select SYMBOLIC_ERRNAME 28 28 select MIN_HEAP 29 + select XARRAY_MULTI 29 30 help 30 31 The bcachefs filesystem - a modern, copy on write filesystem, with 31 32 support for multiple devices, compression, checksumming, etc.

+2 -2

fs/bcachefs/acl.c

··· 273 273 struct bch_fs *c = inode->v.i_sb->s_fs_info; 274 274 struct bch_hash_info hash = bch2_hash_info_init(c, &inode->ei_inode); 275 275 struct xattr_search_key search = X_SEARCH(acl_to_xattr_type(type), "", 0); 276 - struct btree_iter iter = { NULL }; 276 + struct btree_iter iter = {}; 277 277 struct posix_acl *acl = NULL; 278 278 279 279 if (rcu) ··· 344 344 { 345 345 struct bch_inode_info *inode = to_bch_ei(dentry->d_inode); 346 346 struct bch_fs *c = inode->v.i_sb->s_fs_info; 347 - struct btree_iter inode_iter = { NULL }; 347 + struct btree_iter inode_iter = {}; 348 348 struct bch_inode_unpacked inode_u; 349 349 struct posix_acl *acl; 350 350 umode_t mode;

+50 -45

fs/bcachefs/alloc_background.c

··· 610 610 * bch2_check_alloc_key() which runs later: 611 611 */ 612 612 if (!ca) { 613 - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 613 + bch2_btree_iter_set_pos(trans, &iter, POS(k.k->p.inode + 1, 0)); 614 614 continue; 615 615 } 616 616 ··· 631 631 * bch2_check_alloc_key() which runs later: 632 632 */ 633 633 if (!ca) { 634 - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 634 + bch2_btree_iter_set_pos(trans, &iter, POS(k.k->p.inode + 1, 0)); 635 635 continue; 636 636 } 637 637 638 638 if (k.k->p.offset < ca->mi.first_bucket) { 639 - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode, ca->mi.first_bucket)); 639 + bch2_btree_iter_set_pos(trans, &iter, POS(k.k->p.inode, ca->mi.first_bucket)); 640 640 continue; 641 641 } 642 642 643 643 if (k.k->p.offset >= ca->mi.nbuckets) { 644 - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 644 + bch2_btree_iter_set_pos(trans, &iter, POS(k.k->p.inode + 1, 0)); 645 645 continue; 646 646 } 647 647 ··· 1039 1039 * This synthesizes deleted extents for holes, similar to BTREE_ITER_slots for 1040 1040 * extents style btrees, but works on non-extents btrees: 1041 1041 */ 1042 - static struct bkey_s_c bch2_get_key_or_hole(struct btree_iter *iter, struct bpos end, struct bkey *hole) 1042 + static struct bkey_s_c bch2_get_key_or_hole(struct btree_trans *trans, struct btree_iter *iter, 1043 + struct bpos end, struct bkey *hole) 1043 1044 { 1044 - struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); 1045 + struct bkey_s_c k = bch2_btree_iter_peek_slot(trans, iter); 1045 1046 1046 1047 if (bkey_err(k)) 1047 1048 return k; ··· 1053 1052 struct btree_iter iter2; 1054 1053 struct bpos next; 1055 1054 1056 - bch2_trans_copy_iter(&iter2, iter); 1055 + bch2_trans_copy_iter(trans, &iter2, iter); 1057 1056 1058 - struct btree_path *path = btree_iter_path(iter->trans, iter); 1057 + struct btree_path *path = btree_iter_path(trans, iter); 1059 1058 if (!bpos_eq(path->l[0].b->key.k.p, SPOS_MAX)) 1060 1059 end = bkey_min(end, bpos_nosnap_successor(path->l[0].b->key.k.p)); 1061 1060 ··· 1065 1064 * btree node min/max is a closed interval, upto takes a half 1066 1065 * open interval: 1067 1066 */ 1068 - k = bch2_btree_iter_peek_max(&iter2, end); 1067 + k = bch2_btree_iter_peek_max(trans, &iter2, end); 1069 1068 next = iter2.pos; 1070 - bch2_trans_iter_exit(iter->trans, &iter2); 1069 + bch2_trans_iter_exit(trans, &iter2); 1071 1070 1072 1071 BUG_ON(next.offset >= iter->pos.offset + U32_MAX); 1073 1072 ··· 1108 1107 return *ca != NULL; 1109 1108 } 1110 1109 1111 - static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, 1112 - struct bch_dev **ca, struct bkey *hole) 1110 + static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_trans *trans, 1111 + struct btree_iter *iter, 1112 + struct bch_dev **ca, struct bkey *hole) 1113 1113 { 1114 - struct bch_fs *c = iter->trans->c; 1114 + struct bch_fs *c = trans->c; 1115 1115 struct bkey_s_c k; 1116 1116 again: 1117 - k = bch2_get_key_or_hole(iter, POS_MAX, hole); 1117 + k = bch2_get_key_or_hole(trans, iter, POS_MAX, hole); 1118 1118 if (bkey_err(k)) 1119 1119 return k; 1120 1120 ··· 1128 1126 if (!next_bucket(c, ca, &hole_start)) 1129 1127 return bkey_s_c_null; 1130 1128 1131 - bch2_btree_iter_set_pos(iter, hole_start); 1129 + bch2_btree_iter_set_pos(trans, iter, hole_start); 1132 1130 goto again; 1133 1131 } 1134 1132 ··· 1169 1167 1170 1168 a = bch2_alloc_to_v4(alloc_k, &a_convert); 1171 1169 1172 - bch2_btree_iter_set_pos(discard_iter, alloc_k.k->p); 1173 - k = bch2_btree_iter_peek_slot(discard_iter); 1170 + bch2_btree_iter_set_pos(trans, discard_iter, alloc_k.k->p); 1171 + k = bch2_btree_iter_peek_slot(trans, discard_iter); 1174 1172 ret = bkey_err(k); 1175 1173 if (ret) 1176 1174 goto err; ··· 1183 1181 goto err; 1184 1182 } 1185 1183 1186 - bch2_btree_iter_set_pos(freespace_iter, alloc_freespace_pos(alloc_k.k->p, *a)); 1187 - k = bch2_btree_iter_peek_slot(freespace_iter); 1184 + bch2_btree_iter_set_pos(trans, freespace_iter, alloc_freespace_pos(alloc_k.k->p, *a)); 1185 + k = bch2_btree_iter_peek_slot(trans, freespace_iter); 1188 1186 ret = bkey_err(k); 1189 1187 if (ret) 1190 1188 goto err; ··· 1197 1195 goto err; 1198 1196 } 1199 1197 1200 - bch2_btree_iter_set_pos(bucket_gens_iter, alloc_gens_pos(alloc_k.k->p, &gens_offset)); 1201 - k = bch2_btree_iter_peek_slot(bucket_gens_iter); 1198 + bch2_btree_iter_set_pos(trans, bucket_gens_iter, alloc_gens_pos(alloc_k.k->p, &gens_offset)); 1199 + k = bch2_btree_iter_peek_slot(trans, bucket_gens_iter); 1202 1200 ret = bkey_err(k); 1203 1201 if (ret) 1204 1202 goto err; ··· 1251 1249 if (!ca->mi.freespace_initialized) 1252 1250 return 0; 1253 1251 1254 - bch2_btree_iter_set_pos(freespace_iter, start); 1252 + bch2_btree_iter_set_pos(trans, freespace_iter, start); 1255 1253 1256 - k = bch2_btree_iter_peek_slot(freespace_iter); 1254 + k = bch2_btree_iter_peek_slot(trans, freespace_iter); 1257 1255 ret = bkey_err(k); 1258 1256 if (ret) 1259 1257 goto err; ··· 1302 1300 unsigned i, gens_offset, gens_end_offset; 1303 1301 int ret; 1304 1302 1305 - bch2_btree_iter_set_pos(bucket_gens_iter, alloc_gens_pos(start, &gens_offset)); 1303 + bch2_btree_iter_set_pos(trans, bucket_gens_iter, alloc_gens_pos(start, &gens_offset)); 1306 1304 1307 - k = bch2_btree_iter_peek_slot(bucket_gens_iter); 1305 + k = bch2_btree_iter_peek_slot(trans, bucket_gens_iter); 1308 1306 ret = bkey_err(k); 1309 1307 if (ret) 1310 1308 goto err; ··· 1437 1435 *gen = a->gen; 1438 1436 out: 1439 1437 fsck_err: 1440 - bch2_set_btree_iter_dontneed(&alloc_iter); 1438 + bch2_set_btree_iter_dontneed(trans, &alloc_iter); 1441 1439 bch2_trans_iter_exit(trans, &alloc_iter); 1442 1440 printbuf_exit(&buf); 1443 1441 return ret; ··· 1574 1572 1575 1573 bch2_trans_begin(trans); 1576 1574 1577 - k = bch2_get_key_or_real_bucket_hole(&iter, &ca, &hole); 1575 + k = bch2_get_key_or_real_bucket_hole(trans, &iter, &ca, &hole); 1578 1576 ret = bkey_err(k); 1579 1577 if (ret) 1580 1578 goto bkey_err; ··· 1612 1610 if (ret) 1613 1611 goto bkey_err; 1614 1612 1615 - bch2_btree_iter_set_pos(&iter, next); 1613 + bch2_btree_iter_set_pos(trans, &iter, next); 1616 1614 bkey_err: 1617 1615 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 1618 1616 continue; ··· 1640 1638 BTREE_ITER_prefetch); 1641 1639 while (1) { 1642 1640 bch2_trans_begin(trans); 1643 - k = bch2_btree_iter_peek(&iter); 1641 + k = bch2_btree_iter_peek(trans, &iter); 1644 1642 if (!k.k) 1645 1643 break; 1646 1644 ··· 1659 1657 break; 1660 1658 } 1661 1659 1662 - bch2_btree_iter_set_pos(&iter, bpos_nosnap_successor(iter.pos)); 1660 + bch2_btree_iter_set_pos(trans, &iter, bpos_nosnap_successor(iter.pos)); 1663 1661 } 1664 1662 bch2_trans_iter_exit(trans, &iter); 1665 1663 if (ret) ··· 1687 1685 struct printbuf buf = PRINTBUF; 1688 1686 int ret; 1689 1687 1690 - alloc_k = bch2_btree_iter_peek(alloc_iter); 1688 + alloc_k = bch2_btree_iter_peek(trans, alloc_iter); 1691 1689 if (!alloc_k.k) 1692 1690 return 0; 1693 1691 ··· 1828 1826 { 1829 1827 struct bch_fs *c = trans->c; 1830 1828 struct bpos pos = need_discard_iter->pos; 1831 - struct btree_iter iter = { NULL }; 1829 + struct btree_iter iter = {}; 1832 1830 struct bkey_s_c k; 1833 1831 struct bkey_i_alloc_v4 *a; 1834 1832 struct printbuf buf = PRINTBUF; ··· 1952 1950 trace_discard_buckets(c, s.seen, s.open, s.need_journal_commit, s.discarded, 1953 1951 bch2_err_str(ret)); 1954 1952 1955 - percpu_ref_put(&ca->io_ref); 1953 + percpu_ref_put(&ca->io_ref[WRITE]); 1956 1954 bch2_write_ref_put(c, BCH_WRITE_REF_discard); 1957 1955 } 1958 1956 ··· 1969 1967 if (queue_work(c->write_ref_wq, &ca->discard_work)) 1970 1968 return; 1971 1969 1972 - percpu_ref_put(&ca->io_ref); 1970 + percpu_ref_put(&ca->io_ref[WRITE]); 1973 1971 put_write_ref: 1974 1972 bch2_write_ref_put(c, BCH_WRITE_REF_discard); 1975 1973 } ··· 2047 2045 trace_discard_buckets_fast(c, s.seen, s.open, s.need_journal_commit, s.discarded, bch2_err_str(ret)); 2048 2046 2049 2047 bch2_trans_put(trans); 2050 - percpu_ref_put(&ca->io_ref); 2048 + percpu_ref_put(&ca->io_ref[WRITE]); 2051 2049 bch2_write_ref_put(c, BCH_WRITE_REF_discard_fast); 2052 2050 } 2053 2051 ··· 2067 2065 if (queue_work(c->write_ref_wq, &ca->discard_fast_work)) 2068 2066 return; 2069 2067 2070 - percpu_ref_put(&ca->io_ref); 2068 + percpu_ref_put(&ca->io_ref[WRITE]); 2071 2069 put_ref: 2072 2070 bch2_write_ref_put(c, BCH_WRITE_REF_discard_fast); 2073 2071 } ··· 2083 2081 int ret = bkey_err(extent_k); 2084 2082 if (ret) 2085 2083 return ret; 2084 + 2085 + if (!extent_k.k) 2086 + return 0; 2086 2087 2087 2088 struct bkey_i *n = 2088 2089 bch2_bkey_make_mut(trans, &extent_iter, &extent_k, ··· 2204 2199 { 2205 2200 struct bkey_s_c k; 2206 2201 again: 2207 - k = bch2_btree_iter_peek_max(iter, lru_pos(ca->dev_idx, U64_MAX, LRU_TIME_MAX)); 2202 + k = bch2_btree_iter_peek_max(trans, iter, lru_pos(ca->dev_idx, U64_MAX, LRU_TIME_MAX)); 2208 2203 if (!k.k && !*wrapped) { 2209 - bch2_btree_iter_set_pos(iter, lru_pos(ca->dev_idx, 0, 0)); 2204 + bch2_btree_iter_set_pos(trans, iter, lru_pos(ca->dev_idx, 0, 0)); 2210 2205 *wrapped = true; 2211 2206 goto again; 2212 2207 } ··· 2256 2251 if (ret) 2257 2252 break; 2258 2253 2259 - bch2_btree_iter_advance(&iter); 2254 + bch2_btree_iter_advance(trans, &iter); 2260 2255 } 2261 2256 bch2_trans_iter_exit(trans, &iter); 2262 2257 err: 2263 2258 bch2_trans_put(trans); 2264 - percpu_ref_put(&ca->io_ref); 2259 + percpu_ref_put(&ca->io_ref[WRITE]); 2265 2260 bch2_bkey_buf_exit(&last_flushed, c); 2266 2261 bch2_write_ref_put(c, BCH_WRITE_REF_invalidate); 2267 2262 } ··· 2279 2274 if (queue_work(c->write_ref_wq, &ca->invalidate_work)) 2280 2275 return; 2281 2276 2282 - percpu_ref_put(&ca->io_ref); 2277 + percpu_ref_put(&ca->io_ref[WRITE]); 2283 2278 put_ref: 2284 2279 bch2_write_ref_put(c, BCH_WRITE_REF_invalidate); 2285 2280 } ··· 2326 2321 break; 2327 2322 } 2328 2323 2329 - k = bch2_get_key_or_hole(&iter, end, &hole); 2324 + k = bch2_get_key_or_hole(trans, &iter, end, &hole); 2330 2325 ret = bkey_err(k); 2331 2326 if (ret) 2332 2327 goto bkey_err; ··· 2345 2340 if (ret) 2346 2341 goto bkey_err; 2347 2342 2348 - bch2_btree_iter_advance(&iter); 2343 + bch2_btree_iter_advance(trans, &iter); 2349 2344 } else { 2350 2345 struct bkey_i *freespace; 2351 2346 ··· 2365 2360 if (ret) 2366 2361 goto bkey_err; 2367 2362 2368 - bch2_btree_iter_set_pos(&iter, k.k->p); 2363 + bch2_btree_iter_set_pos(trans, &iter, k.k->p); 2369 2364 } 2370 2365 bkey_err: 2371 2366 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) ··· 2511 2506 2512 2507 bch2_set_ra_pages(c, ra_pages); 2513 2508 2514 - for_each_rw_member(c, ca) { 2509 + __for_each_online_member(c, ca, BIT(BCH_MEMBER_STATE_rw), READ) { 2515 2510 u64 dev_reserve = 0; 2516 2511 2517 2512 /*

+3 -3

fs/bcachefs/alloc_background.h

··· 321 321 { 322 322 u64 want_free = ca->mi.nbuckets >> 7; 323 323 u64 free = max_t(s64, 0, 324 - u.d[BCH_DATA_free].buckets 325 - + u.d[BCH_DATA_need_discard].buckets 324 + u.buckets[BCH_DATA_free] 325 + + u.buckets[BCH_DATA_need_discard] 326 326 - bch2_dev_buckets_reserved(ca, BCH_WATERMARK_stripe)); 327 327 328 - return clamp_t(s64, want_free - free, 0, u.d[BCH_DATA_cached].buckets); 328 + return clamp_t(s64, want_free - free, 0, u.buckets[BCH_DATA_cached]); 329 329 } 330 330 331 331 void bch2_dev_do_invalidates(struct bch_dev *);

+58 -21

fs/bcachefs/alloc_foreground.c

··· 327 327 bucket = sector_to_bucket(ca, 328 328 round_up(bucket_to_sector(ca, bucket) + 1, 329 329 1ULL << ca->mi.btree_bitmap_shift)); 330 - bch2_btree_iter_set_pos(&iter, POS(ca->dev_idx, bucket)); 330 + bch2_btree_iter_set_pos(trans, &iter, POS(ca->dev_idx, bucket)); 331 331 s->buckets_seen++; 332 332 s->skipped_mi_btree_bitmap++; 333 333 continue; ··· 355 355 watermark, s, cl) 356 356 : NULL; 357 357 next: 358 - bch2_set_btree_iter_dontneed(&citer); 358 + bch2_set_btree_iter_dontneed(trans, &citer); 359 359 bch2_trans_iter_exit(trans, &citer); 360 360 if (ob) 361 361 break; ··· 417 417 1ULL << ca->mi.btree_bitmap_shift)); 418 418 alloc_cursor = bucket|(iter.pos.offset & (~0ULL << 56)); 419 419 420 - bch2_btree_iter_set_pos(&iter, POS(ca->dev_idx, alloc_cursor)); 420 + bch2_btree_iter_set_pos(trans, &iter, POS(ca->dev_idx, alloc_cursor)); 421 421 s->skipped_mi_btree_bitmap++; 422 422 goto next; 423 423 } ··· 426 426 if (ob) { 427 427 if (!IS_ERR(ob)) 428 428 *dev_alloc_cursor = iter.pos.offset; 429 - bch2_set_btree_iter_dontneed(&iter); 429 + bch2_set_btree_iter_dontneed(trans, &iter); 430 430 break; 431 431 } 432 432 ··· 469 469 prt_printf(&buf, "watermark\t%s\n", bch2_watermarks[watermark]); 470 470 prt_printf(&buf, "data type\t%s\n", __bch2_data_types[data_type]); 471 471 prt_printf(&buf, "blocking\t%u\n", cl != NULL); 472 - prt_printf(&buf, "free\t%llu\n", usage->d[BCH_DATA_free].buckets); 472 + prt_printf(&buf, "free\t%llu\n", usage->buckets[BCH_DATA_free]); 473 473 prt_printf(&buf, "avail\t%llu\n", dev_buckets_free(ca, *usage, watermark)); 474 474 prt_printf(&buf, "copygc_wait\t%lu/%lli\n", 475 475 bch2_copygc_wait_amount(c), ··· 524 524 bch2_dev_usage_read_fast(ca, usage); 525 525 avail = dev_buckets_free(ca, *usage, watermark); 526 526 527 - if (usage->d[BCH_DATA_need_discard].buckets > avail) 527 + if (usage->buckets[BCH_DATA_need_discard] > avail) 528 528 bch2_dev_do_discards(ca); 529 529 530 - if (usage->d[BCH_DATA_need_gc_gens].buckets > avail) 530 + if (usage->buckets[BCH_DATA_need_gc_gens] > avail) 531 531 bch2_gc_gens_async(c); 532 532 533 533 if (should_invalidate_buckets(ca, *usage)) ··· 606 606 static int __dev_stripe_cmp(struct dev_stripe_state *stripe, 607 607 unsigned l, unsigned r) 608 608 { 609 - return ((stripe->next_alloc[l] > stripe->next_alloc[r]) - 610 - (stripe->next_alloc[l] < stripe->next_alloc[r])); 609 + return cmp_int(stripe->next_alloc[l], stripe->next_alloc[r]); 611 610 } 612 611 613 612 #define dev_stripe_cmp(l, r) __dev_stripe_cmp(stripe, l, r) ··· 625 626 return ret; 626 627 } 627 628 629 + static const u64 stripe_clock_hand_rescale = 1ULL << 62; /* trigger rescale at */ 630 + static const u64 stripe_clock_hand_max = 1ULL << 56; /* max after rescale */ 631 + static const u64 stripe_clock_hand_inv = 1ULL << 52; /* max increment, if a device is empty */ 632 + 633 + static noinline void bch2_stripe_state_rescale(struct dev_stripe_state *stripe) 634 + { 635 + /* 636 + * Avoid underflowing clock hands if at all possible, if clock hands go 637 + * to 0 then we lose information - clock hands can be in a wide range if 638 + * we have devices we rarely try to allocate from, if we generally 639 + * allocate from a specified target but only sometimes have to fall back 640 + * to the whole filesystem. 641 + */ 642 + u64 scale_max = U64_MAX; /* maximum we can subtract without underflow */ 643 + u64 scale_min = 0; /* minumum we must subtract to avoid overflow */ 644 + 645 + for (u64 *v = stripe->next_alloc; 646 + v < stripe->next_alloc + ARRAY_SIZE(stripe->next_alloc); v++) { 647 + if (*v) 648 + scale_max = min(scale_max, *v); 649 + if (*v > stripe_clock_hand_max) 650 + scale_min = max(scale_min, *v - stripe_clock_hand_max); 651 + } 652 + 653 + u64 scale = max(scale_min, scale_max); 654 + 655 + for (u64 *v = stripe->next_alloc; 656 + v < stripe->next_alloc + ARRAY_SIZE(stripe->next_alloc); v++) 657 + *v = *v < scale ? 0 : *v - scale; 658 + } 659 + 628 660 static inline void bch2_dev_stripe_increment_inlined(struct bch_dev *ca, 629 661 struct dev_stripe_state *stripe, 630 662 struct bch_dev_usage *usage) 631 663 { 664 + /* 665 + * Stripe state has a per device clock hand: we allocate from the device 666 + * with the smallest clock hand. 667 + * 668 + * When we allocate, we don't do a simple increment; we add the inverse 669 + * of the device's free space. This results in round robin behavior that 670 + * biases in favor of the device(s) with more free space. 671 + */ 672 + 632 673 u64 *v = stripe->next_alloc + ca->dev_idx; 633 674 u64 free_space = __dev_buckets_available(ca, *usage, BCH_WATERMARK_normal); 634 675 u64 free_space_inv = free_space 635 - ? div64_u64(1ULL << 48, free_space) 636 - : 1ULL << 48; 637 - u64 scale = *v / 4; 676 + ? div64_u64(stripe_clock_hand_inv, free_space) 677 + : stripe_clock_hand_inv; 638 678 639 - if (*v + free_space_inv >= *v) 640 - *v += free_space_inv; 641 - else 642 - *v = U64_MAX; 679 + /* Saturating add, avoid overflow: */ 680 + u64 sum = *v + free_space_inv; 681 + *v = sum >= *v ? sum : U64_MAX; 643 682 644 - for (v = stripe->next_alloc; 645 - v < stripe->next_alloc + ARRAY_SIZE(stripe->next_alloc); v++) 646 - *v = *v < scale ? 0 : *v - scale; 683 + if (unlikely(*v > stripe_clock_hand_rescale)) 684 + bch2_stripe_state_rescale(stripe); 647 685 } 648 686 649 687 void bch2_dev_stripe_increment(struct bch_dev *ca, ··· 1669 1633 void bch2_dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) 1670 1634 { 1671 1635 struct bch_fs *c = ca->fs; 1672 - struct bch_dev_usage stats = bch2_dev_usage_read(ca); 1636 + struct bch_dev_usage_full stats = bch2_dev_usage_full_read(ca); 1673 1637 unsigned nr[BCH_DATA_NR]; 1674 1638 1675 1639 memset(nr, 0, sizeof(nr)); ··· 1692 1656 printbuf_tabstop_push(out, 16); 1693 1657 1694 1658 prt_printf(out, "open buckets\t%i\r\n", ca->nr_open_buckets); 1695 - prt_printf(out, "buckets to invalidate\t%llu\r\n", should_invalidate_buckets(ca, stats)); 1659 + prt_printf(out, "buckets to invalidate\t%llu\r\n", 1660 + should_invalidate_buckets(ca, bch2_dev_usage_read(ca))); 1696 1661 } 1697 1662 1698 1663 static noinline void bch2_print_allocator_stuck(struct bch_fs *c)

+18 -6

fs/bcachefs/backpointers.c

··· 252 252 0, 253 253 bp.v->level, 254 254 iter_flags); 255 - struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); 255 + struct bkey_s_c k = bch2_btree_iter_peek_slot(trans, iter); 256 256 if (bkey_err(k)) { 257 257 bch2_trans_iter_exit(trans, iter); 258 258 return k; 259 + } 260 + 261 + /* 262 + * peek_slot() doesn't normally return NULL - except when we ask for a 263 + * key at a btree level that doesn't exist. 264 + * 265 + * We may want to revisit this and change peek_slot(): 266 + */ 267 + if (!k.k) { 268 + bkey_init(&iter->k); 269 + iter->k.p = bp.v->pos; 270 + k.k = &iter->k; 259 271 } 260 272 261 273 if (k.k && ··· 305 293 0, 306 294 bp.v->level - 1, 307 295 0); 308 - struct btree *b = bch2_btree_iter_peek_node(iter); 296 + struct btree *b = bch2_btree_iter_peek_node(trans, iter); 309 297 if (IS_ERR_OR_NULL(b)) 310 298 goto err; 311 299 ··· 333 321 return 0; 334 322 335 323 struct bch_fs *c = trans->c; 336 - struct btree_iter alloc_iter = { NULL }; 324 + struct btree_iter alloc_iter = {}; 337 325 struct bkey_s_c alloc_k; 338 326 struct printbuf buf = PRINTBUF; 339 327 int ret = 0; ··· 474 462 if (bio) 475 463 bio_put(bio); 476 464 kvfree(data_buf); 477 - percpu_ref_put(&ca->io_ref); 465 + percpu_ref_put(&ca->io_ref[READ]); 478 466 printbuf_exit(&buf); 479 467 return ret; 480 468 } ··· 662 650 retry: 663 651 bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 664 652 0, bch2_btree_id_root(c, btree_id)->b->c.level, 0); 665 - b = bch2_btree_iter_peek_node(&iter); 653 + b = bch2_btree_iter_peek_node(trans, &iter); 666 654 ret = PTR_ERR_OR_ZERO(b); 667 655 if (ret) 668 656 goto err; ··· 946 934 { 947 935 struct btree_iter iter; 948 936 bch2_trans_node_iter_init(trans, &iter, btree, k->k.p, 0, level, 0); 949 - struct btree *b = bch2_btree_iter_peek_node(&iter); 937 + struct btree *b = bch2_btree_iter_peek_node(trans, &iter); 950 938 int ret = PTR_ERR_OR_ZERO(b); 951 939 if (ret) 952 940 goto err;

+4 -3

fs/bcachefs/bcachefs.h

··· 524 524 struct percpu_ref ref; 525 525 #endif 526 526 struct completion ref_completion; 527 - struct percpu_ref io_ref; 528 - struct completion io_ref_completion; 527 + struct percpu_ref io_ref[2]; 528 + struct completion io_ref_completion[2]; 529 529 530 530 struct bch_fs *fs; 531 531 ··· 562 562 unsigned long *bucket_backpointer_mismatches; 563 563 unsigned long *bucket_backpointer_empty; 564 564 565 - struct bch_dev_usage __percpu *usage; 565 + struct bch_dev_usage_full __percpu 566 + *usage; 566 567 567 568 /* Allocator: */ 568 569 u64 alloc_cursor[3];

+2 -2

fs/bcachefs/btree_gc.c

··· 691 691 struct btree_iter iter; 692 692 bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 693 693 0, bch2_btree_id_root(c, btree)->b->c.level, 0); 694 - struct btree *b = bch2_btree_iter_peek_node(&iter); 694 + struct btree *b = bch2_btree_iter_peek_node(trans, &iter); 695 695 ret = PTR_ERR_OR_ZERO(b); 696 696 if (ret) 697 697 goto err_root; ··· 1199 1199 BCH_TRANS_COMMIT_no_enospc, ({ 1200 1200 ca = bch2_dev_iterate(c, ca, k.k->p.inode); 1201 1201 if (!ca) { 1202 - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 1202 + bch2_btree_iter_set_pos(trans, &iter, POS(k.k->p.inode + 1, 0)); 1203 1203 continue; 1204 1204 } 1205 1205 bch2_alloc_write_oldest_gen(trans, ca, &iter, k);

+12 -5

fs/bcachefs/btree_io.c

··· 1353 1353 "btree read error %s for %s", 1354 1354 bch2_blk_status_to_str(bio->bi_status), buf.buf); 1355 1355 if (rb->have_ioref) 1356 - percpu_ref_put(&ca->io_ref); 1356 + percpu_ref_put(&ca->io_ref[READ]); 1357 1357 rb->have_ioref = false; 1358 1358 1359 1359 bch2_mark_io_failure(&failed, &rb->pick, false); ··· 1609 1609 struct bch_dev *ca = bch2_dev_have_ref(c, rb->pick.ptr.dev); 1610 1610 1611 1611 bch2_latency_acct(ca, rb->start_time, READ); 1612 + percpu_ref_put(&ca->io_ref[READ]); 1612 1613 } 1613 1614 1614 1615 ra->err[rb->idx] = bio->bi_status; ··· 1909 1908 scrub->key.k->k.p, 0, scrub->level - 1, 0); 1910 1909 1911 1910 struct btree *b; 1912 - int ret = lockrestart_do(trans, PTR_ERR_OR_ZERO(b = bch2_btree_iter_peek_node(&iter))); 1911 + int ret = lockrestart_do(trans, 1912 + PTR_ERR_OR_ZERO(b = bch2_btree_iter_peek_node(trans, &iter))); 1913 1913 if (ret) 1914 1914 goto err; 1915 1915 ··· 1929 1927 printbuf_exit(&err); 1930 1928 bch2_bkey_buf_exit(&scrub->key, c);; 1931 1929 btree_bounce_free(c, c->opts.btree_node_size, scrub->used_mempool, scrub->buf); 1932 - percpu_ref_put(&scrub->ca->io_ref); 1930 + percpu_ref_put(&scrub->ca->io_ref[READ]); 1933 1931 kfree(scrub); 1934 1932 bch2_write_ref_put(c, BCH_WRITE_REF_btree_node_scrub); 1935 1933 } ··· 1998 1996 return 0; 1999 1997 err_free: 2000 1998 btree_bounce_free(c, c->opts.btree_node_size, used_mempool, buf); 2001 - percpu_ref_put(&ca->io_ref); 1999 + percpu_ref_put(&ca->io_ref[READ]); 2002 2000 err: 2003 2001 bch2_write_ref_put(c, BCH_WRITE_REF_btree_node_scrub); 2004 2002 return ret; ··· 2146 2144 2147 2145 if (ca && bio->bi_status) { 2148 2146 struct printbuf buf = PRINTBUF; 2147 + buf.atomic++; 2149 2148 prt_printf(&buf, "btree write error: %s\n ", 2150 2149 bch2_blk_status_to_str(bio->bi_status)); 2151 2150 bch2_btree_pos_to_text(&buf, c, b); ··· 2161 2158 spin_unlock_irqrestore(&c->btree_write_error_lock, flags); 2162 2159 } 2163 2160 2161 + /* 2162 + * XXX: we should be using io_ref[WRITE], but we aren't retrying failed 2163 + * btree writes yet (due to device removal/ro): 2164 + */ 2164 2165 if (wbio->have_ioref) 2165 - percpu_ref_put(&ca->io_ref); 2166 + percpu_ref_put(&ca->io_ref[READ]); 2166 2167 2167 2168 if (parent) { 2168 2169 bio_put(bio);

+92 -96

fs/bcachefs/btree_iter.c

··· 244 244 bch2_btree_path_verify(trans, path); 245 245 } 246 246 247 - static void bch2_btree_iter_verify(struct btree_iter *iter) 247 + static void bch2_btree_iter_verify(struct btree_trans *trans, struct btree_iter *iter) 248 248 { 249 - struct btree_trans *trans = iter->trans; 250 - 251 249 BUG_ON(!!(iter->flags & BTREE_ITER_cached) != btree_iter_path(trans, iter)->cached); 252 250 253 251 BUG_ON((iter->flags & BTREE_ITER_is_extents) && ··· 274 276 bkey_gt(iter->pos, iter->k.p))); 275 277 } 276 278 277 - static int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k) 279 + static int bch2_btree_iter_verify_ret(struct btree_trans *trans, 280 + struct btree_iter *iter, struct bkey_s_c k) 278 281 { 279 - struct btree_trans *trans = iter->trans; 280 282 struct btree_iter copy; 281 283 struct bkey_s_c prev; 282 284 int ret = 0; ··· 297 299 bch2_trans_iter_init(trans, &copy, iter->btree_id, iter->pos, 298 300 BTREE_ITER_nopreserve| 299 301 BTREE_ITER_all_snapshots); 300 - prev = bch2_btree_iter_prev(&copy); 302 + prev = bch2_btree_iter_prev(trans, &copy); 301 303 if (!prev.k) 302 304 goto out; 303 305 ··· 363 365 struct btree_path *path, unsigned l) {} 364 366 static inline void bch2_btree_path_verify(struct btree_trans *trans, 365 367 struct btree_path *path) {} 366 - static inline void bch2_btree_iter_verify(struct btree_iter *iter) {} 368 + static inline void bch2_btree_iter_verify(struct btree_trans *trans, 369 + struct btree_iter *iter) {} 367 370 static inline void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) {} 368 - static inline int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k) { return 0; } 371 + static inline int bch2_btree_iter_verify_ret(struct btree_trans *trans, struct btree_iter *iter, 372 + struct bkey_s_c k) { return 0; } 369 373 370 374 #endif 371 375 ··· 1855 1855 return (struct bkey_s_c) { u, NULL }; 1856 1856 } 1857 1857 1858 - void bch2_set_btree_iter_dontneed(struct btree_iter *iter) 1858 + void bch2_set_btree_iter_dontneed(struct btree_trans *trans, struct btree_iter *iter) 1859 1859 { 1860 - struct btree_trans *trans = iter->trans; 1861 - 1862 1860 if (!iter->path || trans->restarted) 1863 1861 return; 1864 1862 ··· 1868 1870 /* Btree iterators: */ 1869 1871 1870 1872 int __must_check 1871 - __bch2_btree_iter_traverse(struct btree_iter *iter) 1873 + __bch2_btree_iter_traverse(struct btree_trans *trans, struct btree_iter *iter) 1872 1874 { 1873 - return bch2_btree_path_traverse(iter->trans, iter->path, iter->flags); 1875 + return bch2_btree_path_traverse(trans, iter->path, iter->flags); 1874 1876 } 1875 1877 1876 1878 int __must_check 1877 - bch2_btree_iter_traverse(struct btree_iter *iter) 1879 + bch2_btree_iter_traverse(struct btree_trans *trans, struct btree_iter *iter) 1878 1880 { 1879 - struct btree_trans *trans = iter->trans; 1880 - int ret; 1881 - 1882 1881 bch2_trans_verify_not_unlocked_or_in_restart(trans); 1883 1882 1884 1883 iter->path = bch2_btree_path_set_pos(trans, iter->path, ··· 1883 1888 iter->flags & BTREE_ITER_intent, 1884 1889 btree_iter_ip_allocated(iter)); 1885 1890 1886 - ret = bch2_btree_path_traverse(iter->trans, iter->path, iter->flags); 1891 + int ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); 1887 1892 if (ret) 1888 1893 return ret; 1889 1894 ··· 1895 1900 1896 1901 /* Iterate across nodes (leaf and interior nodes) */ 1897 1902 1898 - struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) 1903 + struct btree *bch2_btree_iter_peek_node(struct btree_trans *trans, 1904 + struct btree_iter *iter) 1899 1905 { 1900 - struct btree_trans *trans = iter->trans; 1901 1906 struct btree *b = NULL; 1902 1907 int ret; 1903 1908 1904 1909 EBUG_ON(trans->paths[iter->path].cached); 1905 - bch2_btree_iter_verify(iter); 1910 + bch2_btree_iter_verify(trans, iter); 1906 1911 1907 1912 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); 1908 1913 if (ret) ··· 1924 1929 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); 1925 1930 out: 1926 1931 bch2_btree_iter_verify_entry_exit(iter); 1927 - bch2_btree_iter_verify(iter); 1932 + bch2_btree_iter_verify(trans, iter); 1928 1933 1929 1934 return b; 1930 1935 err: ··· 1933 1938 } 1934 1939 1935 1940 /* Only kept for -tools */ 1936 - struct btree *bch2_btree_iter_peek_node_and_restart(struct btree_iter *iter) 1941 + struct btree *bch2_btree_iter_peek_node_and_restart(struct btree_trans *trans, 1942 + struct btree_iter *iter) 1937 1943 { 1938 1944 struct btree *b; 1939 1945 1940 - while (b = bch2_btree_iter_peek_node(iter), 1946 + while (b = bch2_btree_iter_peek_node(trans, iter), 1941 1947 bch2_err_matches(PTR_ERR_OR_ZERO(b), BCH_ERR_transaction_restart)) 1942 - bch2_trans_begin(iter->trans); 1948 + bch2_trans_begin(trans); 1943 1949 1944 1950 return b; 1945 1951 } 1946 1952 1947 - struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) 1953 + struct btree *bch2_btree_iter_next_node(struct btree_trans *trans, struct btree_iter *iter) 1948 1954 { 1949 - struct btree_trans *trans = iter->trans; 1950 1955 struct btree *b = NULL; 1951 1956 int ret; 1952 1957 1953 1958 EBUG_ON(trans->paths[iter->path].cached); 1954 1959 bch2_trans_verify_not_unlocked_or_in_restart(trans); 1955 - bch2_btree_iter_verify(iter); 1960 + bch2_btree_iter_verify(trans, iter); 1956 1961 1957 1962 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); 1958 1963 if (ret) ··· 2019 2024 EBUG_ON(btree_iter_path(trans, iter)->uptodate); 2020 2025 out: 2021 2026 bch2_btree_iter_verify_entry_exit(iter); 2022 - bch2_btree_iter_verify(iter); 2027 + bch2_btree_iter_verify(trans, iter); 2023 2028 2024 2029 return b; 2025 2030 err: ··· 2029 2034 2030 2035 /* Iterate across keys (in leaf nodes only) */ 2031 2036 2032 - inline bool bch2_btree_iter_advance(struct btree_iter *iter) 2037 + inline bool bch2_btree_iter_advance(struct btree_trans *trans, struct btree_iter *iter) 2033 2038 { 2034 2039 struct bpos pos = iter->k.p; 2035 2040 bool ret = !(iter->flags & BTREE_ITER_all_snapshots ··· 2038 2043 2039 2044 if (ret && !(iter->flags & BTREE_ITER_is_extents)) 2040 2045 pos = bkey_successor(iter, pos); 2041 - bch2_btree_iter_set_pos(iter, pos); 2046 + bch2_btree_iter_set_pos(trans, iter, pos); 2042 2047 return ret; 2043 2048 } 2044 2049 2045 - inline bool bch2_btree_iter_rewind(struct btree_iter *iter) 2050 + inline bool bch2_btree_iter_rewind(struct btree_trans *trans, struct btree_iter *iter) 2046 2051 { 2047 2052 struct bpos pos = bkey_start_pos(&iter->k); 2048 2053 bool ret = !(iter->flags & BTREE_ITER_all_snapshots ··· 2051 2056 2052 2057 if (ret && !(iter->flags & BTREE_ITER_is_extents)) 2053 2058 pos = bkey_predecessor(iter, pos); 2054 - bch2_btree_iter_set_pos(iter, pos); 2059 + bch2_btree_iter_set_pos(trans, iter, pos); 2055 2060 return ret; 2056 2061 } 2057 2062 ··· 2178 2183 * bkey_s_c_null: 2179 2184 */ 2180 2185 static noinline 2181 - struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos pos) 2186 + struct bkey_s_c btree_trans_peek_key_cache(struct btree_trans *trans, struct btree_iter *iter, 2187 + struct bpos pos) 2182 2188 { 2183 - struct btree_trans *trans = iter->trans; 2184 2189 struct bch_fs *c = trans->c; 2185 2190 struct bkey u; 2186 2191 struct bkey_s_c k; ··· 2226 2231 return k; 2227 2232 } 2228 2233 2229 - static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bpos search_key) 2234 + static struct bkey_s_c __bch2_btree_iter_peek(struct btree_trans *trans, struct btree_iter *iter, 2235 + struct bpos search_key) 2230 2236 { 2231 - struct btree_trans *trans = iter->trans; 2232 2237 struct bkey_s_c k, k2; 2233 2238 int ret; 2234 2239 2235 2240 EBUG_ON(btree_iter_path(trans, iter)->cached); 2236 - bch2_btree_iter_verify(iter); 2241 + bch2_btree_iter_verify(trans, iter); 2237 2242 2238 2243 while (1) { 2239 2244 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, ··· 2243 2248 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); 2244 2249 if (unlikely(ret)) { 2245 2250 /* ensure that iter->k is consistent with iter->pos: */ 2246 - bch2_btree_iter_set_pos(iter, iter->pos); 2251 + bch2_btree_iter_set_pos(trans, iter, iter->pos); 2247 2252 k = bkey_s_c_err(ret); 2248 2253 break; 2249 2254 } ··· 2253 2258 2254 2259 if (unlikely(!l->b)) { 2255 2260 /* No btree nodes at requested level: */ 2256 - bch2_btree_iter_set_pos(iter, SPOS_MAX); 2261 + bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); 2257 2262 k = bkey_s_c_null; 2258 2263 break; 2259 2264 } ··· 2264 2269 2265 2270 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && 2266 2271 k.k && 2267 - (k2 = btree_trans_peek_key_cache(iter, k.k->p)).k) { 2272 + (k2 = btree_trans_peek_key_cache(trans, iter, k.k->p)).k) { 2268 2273 k = k2; 2269 2274 if (bkey_err(k)) { 2270 - bch2_btree_iter_set_pos(iter, iter->pos); 2275 + bch2_btree_iter_set_pos(trans, iter, iter->pos); 2271 2276 break; 2272 2277 } 2273 2278 } ··· 2300 2305 search_key = bpos_successor(l->b->key.k.p); 2301 2306 } else { 2302 2307 /* End of btree: */ 2303 - bch2_btree_iter_set_pos(iter, SPOS_MAX); 2308 + bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); 2304 2309 k = bkey_s_c_null; 2305 2310 break; 2306 2311 } 2307 2312 } 2308 2313 2309 - bch2_btree_iter_verify(iter); 2314 + bch2_btree_iter_verify(trans, iter); 2310 2315 return k; 2311 2316 } 2312 2317 2313 2318 /** 2314 2319 * bch2_btree_iter_peek_max() - returns first key greater than or equal to 2315 2320 * iterator's current position 2321 + * @trans: btree transaction object 2316 2322 * @iter: iterator to peek from 2317 2323 * @end: search limit: returns keys less than or equal to @end 2318 2324 * 2319 2325 * Returns: key if found, or an error extractable with bkey_err(). 2320 2326 */ 2321 - struct bkey_s_c bch2_btree_iter_peek_max(struct btree_iter *iter, struct bpos end) 2327 + struct bkey_s_c bch2_btree_iter_peek_max(struct btree_trans *trans, struct btree_iter *iter, 2328 + struct bpos end) 2322 2329 { 2323 - struct btree_trans *trans = iter->trans; 2324 2330 struct bpos search_key = btree_iter_search_key(iter); 2325 2331 struct bkey_s_c k; 2326 2332 struct bpos iter_pos = iter->pos; ··· 2344 2348 } 2345 2349 2346 2350 while (1) { 2347 - k = __bch2_btree_iter_peek(iter, search_key); 2351 + k = __bch2_btree_iter_peek(trans, iter, search_key); 2348 2352 if (unlikely(!k.k)) 2349 2353 goto end; 2350 2354 if (unlikely(bkey_err(k))) ··· 2458 2462 if (!(iter->flags & BTREE_ITER_all_snapshots)) 2459 2463 iter->pos.snapshot = iter->snapshot; 2460 2464 2461 - ret = bch2_btree_iter_verify_ret(iter, k); 2465 + ret = bch2_btree_iter_verify_ret(trans, iter, k); 2462 2466 if (unlikely(ret)) { 2463 - bch2_btree_iter_set_pos(iter, iter->pos); 2467 + bch2_btree_iter_set_pos(trans, iter, iter->pos); 2464 2468 k = bkey_s_c_err(ret); 2465 2469 } 2466 2470 ··· 2468 2472 2469 2473 return k; 2470 2474 end: 2471 - bch2_btree_iter_set_pos(iter, end); 2475 + bch2_btree_iter_set_pos(trans, iter, end); 2472 2476 k = bkey_s_c_null; 2473 2477 goto out_no_locked; 2474 2478 } ··· 2476 2480 /** 2477 2481 * bch2_btree_iter_next() - returns first key greater than iterator's current 2478 2482 * position 2483 + * @trans: btree transaction object 2479 2484 * @iter: iterator to peek from 2480 2485 * 2481 2486 * Returns: key if found, or an error extractable with bkey_err(). 2482 2487 */ 2483 - struct bkey_s_c bch2_btree_iter_next(struct btree_iter *iter) 2488 + struct bkey_s_c bch2_btree_iter_next(struct btree_trans *trans, struct btree_iter *iter) 2484 2489 { 2485 - if (!bch2_btree_iter_advance(iter)) 2490 + if (!bch2_btree_iter_advance(trans, iter)) 2486 2491 return bkey_s_c_null; 2487 2492 2488 - return bch2_btree_iter_peek(iter); 2493 + return bch2_btree_iter_peek(trans, iter); 2489 2494 } 2490 2495 2491 - static struct bkey_s_c __bch2_btree_iter_peek_prev(struct btree_iter *iter, struct bpos search_key) 2496 + static struct bkey_s_c __bch2_btree_iter_peek_prev(struct btree_trans *trans, struct btree_iter *iter, 2497 + struct bpos search_key) 2492 2498 { 2493 - struct btree_trans *trans = iter->trans; 2494 2499 struct bkey_s_c k, k2; 2495 2500 2496 - bch2_btree_iter_verify(iter); 2501 + bch2_btree_iter_verify(trans, iter); 2497 2502 2498 2503 while (1) { 2499 2504 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, ··· 2504 2507 int ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); 2505 2508 if (unlikely(ret)) { 2506 2509 /* ensure that iter->k is consistent with iter->pos: */ 2507 - bch2_btree_iter_set_pos(iter, iter->pos); 2510 + bch2_btree_iter_set_pos(trans, iter, iter->pos); 2508 2511 k = bkey_s_c_err(ret); 2509 2512 break; 2510 2513 } ··· 2514 2517 2515 2518 if (unlikely(!l->b)) { 2516 2519 /* No btree nodes at requested level: */ 2517 - bch2_btree_iter_set_pos(iter, SPOS_MAX); 2520 + bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); 2518 2521 k = bkey_s_c_null; 2519 2522 break; 2520 2523 } ··· 2530 2533 2531 2534 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && 2532 2535 k.k && 2533 - (k2 = btree_trans_peek_key_cache(iter, k.k->p)).k) { 2536 + (k2 = btree_trans_peek_key_cache(trans, iter, k.k->p)).k) { 2534 2537 k = k2; 2535 2538 if (bkey_err(k2)) { 2536 - bch2_btree_iter_set_pos(iter, iter->pos); 2539 + bch2_btree_iter_set_pos(trans, iter, iter->pos); 2537 2540 break; 2538 2541 } 2539 2542 } ··· 2554 2557 search_key = bpos_predecessor(path->l[0].b->data->min_key); 2555 2558 } else { 2556 2559 /* Start of btree: */ 2557 - bch2_btree_iter_set_pos(iter, POS_MIN); 2560 + bch2_btree_iter_set_pos(trans, iter, POS_MIN); 2558 2561 k = bkey_s_c_null; 2559 2562 break; 2560 2563 } 2561 2564 } 2562 2565 2563 - bch2_btree_iter_verify(iter); 2566 + bch2_btree_iter_verify(trans, iter); 2564 2567 return k; 2565 2568 } 2566 2569 2567 2570 /** 2568 2571 * bch2_btree_iter_peek_prev_min() - returns first key less than or equal to 2569 2572 * iterator's current position 2573 + * @trans: btree transaction object 2570 2574 * @iter: iterator to peek from 2571 2575 * @end: search limit: returns keys greater than or equal to @end 2572 2576 * 2573 2577 * Returns: key if found, or an error extractable with bkey_err(). 2574 2578 */ 2575 - struct bkey_s_c bch2_btree_iter_peek_prev_min(struct btree_iter *iter, struct bpos end) 2579 + struct bkey_s_c bch2_btree_iter_peek_prev_min(struct btree_trans *trans, struct btree_iter *iter, 2580 + struct bpos end) 2576 2581 { 2577 2582 if ((iter->flags & (BTREE_ITER_is_extents|BTREE_ITER_filter_snapshots)) && 2578 2583 !bkey_eq(iter->pos, POS_MAX)) { ··· 2586 2587 * real visible extents - easiest to just use peek_slot() (which 2587 2588 * internally uses peek() for extents) 2588 2589 */ 2589 - struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); 2590 + struct bkey_s_c k = bch2_btree_iter_peek_slot(trans, iter); 2590 2591 if (bkey_err(k)) 2591 2592 return k; 2592 2593 ··· 2596 2597 return k; 2597 2598 } 2598 2599 2599 - struct btree_trans *trans = iter->trans; 2600 2600 struct bpos search_key = iter->pos; 2601 2601 struct bkey_s_c k; 2602 2602 btree_path_idx_t saved_path = 0; ··· 2611 2613 } 2612 2614 2613 2615 while (1) { 2614 - k = __bch2_btree_iter_peek_prev(iter, search_key); 2616 + k = __bch2_btree_iter_peek_prev(trans, iter, search_key); 2615 2617 if (unlikely(!k.k)) 2616 2618 goto end; 2617 2619 if (unlikely(bkey_err(k))) ··· 2702 2704 bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_intent); 2703 2705 2704 2706 bch2_btree_iter_verify_entry_exit(iter); 2705 - bch2_btree_iter_verify(iter); 2707 + bch2_btree_iter_verify(trans, iter); 2706 2708 return k; 2707 2709 end: 2708 - bch2_btree_iter_set_pos(iter, end); 2710 + bch2_btree_iter_set_pos(trans, iter, end); 2709 2711 k = bkey_s_c_null; 2710 2712 goto out_no_locked; 2711 2713 } ··· 2713 2715 /** 2714 2716 * bch2_btree_iter_prev() - returns first key less than iterator's current 2715 2717 * position 2718 + * @trans: btree transaction object 2716 2719 * @iter: iterator to peek from 2717 2720 * 2718 2721 * Returns: key if found, or an error extractable with bkey_err(). 2719 2722 */ 2720 - struct bkey_s_c bch2_btree_iter_prev(struct btree_iter *iter) 2723 + struct bkey_s_c bch2_btree_iter_prev(struct btree_trans *trans, struct btree_iter *iter) 2721 2724 { 2722 - if (!bch2_btree_iter_rewind(iter)) 2725 + if (!bch2_btree_iter_rewind(trans, iter)) 2723 2726 return bkey_s_c_null; 2724 2727 2725 - return bch2_btree_iter_peek_prev(iter); 2728 + return bch2_btree_iter_peek_prev(trans, iter); 2726 2729 } 2727 2730 2728 - struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) 2731 + struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_trans *trans, struct btree_iter *iter) 2729 2732 { 2730 - struct btree_trans *trans = iter->trans; 2731 2733 struct bpos search_key; 2732 2734 struct bkey_s_c k; 2733 2735 int ret; 2734 2736 2735 2737 bch2_trans_verify_not_unlocked_or_in_restart(trans); 2736 - bch2_btree_iter_verify(iter); 2738 + bch2_btree_iter_verify(trans, iter); 2737 2739 bch2_btree_iter_verify_entry_exit(iter); 2738 2740 EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_with_key_cache)); 2739 2741 ··· 2749 2751 if (iter->pos.inode == KEY_INODE_MAX) 2750 2752 return bkey_s_c_null; 2751 2753 2752 - bch2_btree_iter_set_pos(iter, bpos_nosnap_successor(iter->pos)); 2754 + bch2_btree_iter_set_pos(trans, iter, bpos_nosnap_successor(iter->pos)); 2753 2755 } 2754 2756 2755 2757 search_key = btree_iter_search_key(iter); ··· 2783 2785 goto out; 2784 2786 2785 2787 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && 2786 - (k = btree_trans_peek_key_cache(iter, iter->pos)).k) { 2788 + (k = btree_trans_peek_key_cache(trans, iter, iter->pos)).k) { 2787 2789 if (!bkey_err(k)) 2788 2790 iter->k = *k.k; 2789 2791 /* We're not returning a key from iter->path: */ ··· 2810 2812 if (iter->flags & BTREE_ITER_intent) { 2811 2813 struct btree_iter iter2; 2812 2814 2813 - bch2_trans_copy_iter(&iter2, iter); 2814 - k = bch2_btree_iter_peek_max(&iter2, end); 2815 + bch2_trans_copy_iter(trans, &iter2, iter); 2816 + k = bch2_btree_iter_peek_max(trans, &iter2, end); 2815 2817 2816 2818 if (k.k && !bkey_err(k)) { 2817 2819 swap(iter->key_cache_path, iter2.key_cache_path); ··· 2822 2824 } else { 2823 2825 struct bpos pos = iter->pos; 2824 2826 2825 - k = bch2_btree_iter_peek_max(iter, end); 2827 + k = bch2_btree_iter_peek_max(trans, iter, end); 2826 2828 if (unlikely(bkey_err(k))) 2827 - bch2_btree_iter_set_pos(iter, pos); 2829 + bch2_btree_iter_set_pos(trans, iter, pos); 2828 2830 else 2829 2831 iter->pos = pos; 2830 2832 } ··· 2855 2857 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); 2856 2858 out_no_locked: 2857 2859 bch2_btree_iter_verify_entry_exit(iter); 2858 - bch2_btree_iter_verify(iter); 2859 - ret = bch2_btree_iter_verify_ret(iter, k); 2860 + bch2_btree_iter_verify(trans, iter); 2861 + ret = bch2_btree_iter_verify_ret(trans, iter, k); 2860 2862 if (unlikely(ret)) 2861 2863 return bkey_s_c_err(ret); 2862 2864 2863 2865 return k; 2864 2866 } 2865 2867 2866 - struct bkey_s_c bch2_btree_iter_next_slot(struct btree_iter *iter) 2868 + struct bkey_s_c bch2_btree_iter_next_slot(struct btree_trans *trans, struct btree_iter *iter) 2867 2869 { 2868 - if (!bch2_btree_iter_advance(iter)) 2870 + if (!bch2_btree_iter_advance(trans, iter)) 2869 2871 return bkey_s_c_null; 2870 2872 2871 - return bch2_btree_iter_peek_slot(iter); 2873 + return bch2_btree_iter_peek_slot(trans, iter); 2872 2874 } 2873 2875 2874 - struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_iter *iter) 2876 + struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_trans *trans, struct btree_iter *iter) 2875 2877 { 2876 - if (!bch2_btree_iter_rewind(iter)) 2878 + if (!bch2_btree_iter_rewind(trans, iter)) 2877 2879 return bkey_s_c_null; 2878 2880 2879 - return bch2_btree_iter_peek_slot(iter); 2881 + return bch2_btree_iter_peek_slot(trans, iter); 2880 2882 } 2881 2883 2882 2884 /* Obsolete, but still used by rust wrapper in -tools */ 2883 - struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *iter) 2885 + struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_trans *trans, struct btree_iter *iter) 2884 2886 { 2885 2887 struct bkey_s_c k; 2886 2888 2887 - while (btree_trans_too_many_iters(iter->trans) || 2888 - (k = bch2_btree_iter_peek_type(iter, iter->flags), 2889 + while (btree_trans_too_many_iters(trans) || 2890 + (k = bch2_btree_iter_peek_type(trans, iter, iter->flags), 2889 2891 bch2_err_matches(bkey_err(k), BCH_ERR_transaction_restart))) 2890 - bch2_trans_begin(iter->trans); 2892 + bch2_trans_begin(trans); 2891 2893 2892 2894 return k; 2893 2895 } ··· 3033 3035 iter->path = 0; 3034 3036 iter->update_path = 0; 3035 3037 iter->key_cache_path = 0; 3036 - iter->trans = NULL; 3037 3038 } 3038 3039 3039 3040 void bch2_trans_iter_init_outlined(struct btree_trans *trans, ··· 3072 3075 BUG_ON(iter->min_depth != depth); 3073 3076 } 3074 3077 3075 - void bch2_trans_copy_iter(struct btree_iter *dst, struct btree_iter *src) 3078 + void bch2_trans_copy_iter(struct btree_trans *trans, 3079 + struct btree_iter *dst, struct btree_iter *src) 3076 3080 { 3077 - struct btree_trans *trans = src->trans; 3078 - 3079 3081 *dst = *src; 3080 3082 #ifdef TRACK_PATH_ALLOCATED 3081 3083 dst->ip_allocated = _RET_IP_;

+63 -59

fs/bcachefs/btree_iter.h

··· 393 393 void bch2_trans_node_drop(struct btree_trans *trans, struct btree *); 394 394 void bch2_trans_node_reinit_iter(struct btree_trans *, struct btree *); 395 395 396 - int __must_check __bch2_btree_iter_traverse(struct btree_iter *iter); 397 - int __must_check bch2_btree_iter_traverse(struct btree_iter *); 396 + int __must_check __bch2_btree_iter_traverse(struct btree_trans *, struct btree_iter *); 397 + int __must_check bch2_btree_iter_traverse(struct btree_trans *, struct btree_iter *); 398 398 399 - struct btree *bch2_btree_iter_peek_node(struct btree_iter *); 400 - struct btree *bch2_btree_iter_peek_node_and_restart(struct btree_iter *); 401 - struct btree *bch2_btree_iter_next_node(struct btree_iter *); 399 + struct btree *bch2_btree_iter_peek_node(struct btree_trans *, struct btree_iter *); 400 + struct btree *bch2_btree_iter_peek_node_and_restart(struct btree_trans *, struct btree_iter *); 401 + struct btree *bch2_btree_iter_next_node(struct btree_trans *, struct btree_iter *); 402 402 403 - struct bkey_s_c bch2_btree_iter_peek_max(struct btree_iter *, struct bpos); 404 - struct bkey_s_c bch2_btree_iter_next(struct btree_iter *); 403 + struct bkey_s_c bch2_btree_iter_peek_max(struct btree_trans *, struct btree_iter *, struct bpos); 404 + struct bkey_s_c bch2_btree_iter_next(struct btree_trans *, struct btree_iter *); 405 405 406 - static inline struct bkey_s_c bch2_btree_iter_peek(struct btree_iter *iter) 406 + static inline struct bkey_s_c bch2_btree_iter_peek(struct btree_trans *trans, 407 + struct btree_iter *iter) 407 408 { 408 - return bch2_btree_iter_peek_max(iter, SPOS_MAX); 409 + return bch2_btree_iter_peek_max(trans, iter, SPOS_MAX); 409 410 } 410 411 411 - struct bkey_s_c bch2_btree_iter_peek_prev_min(struct btree_iter *, struct bpos); 412 + struct bkey_s_c bch2_btree_iter_peek_prev_min(struct btree_trans *, struct btree_iter *, struct bpos); 412 413 413 - static inline struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) 414 + static inline struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_trans *trans, struct btree_iter *iter) 414 415 { 415 - return bch2_btree_iter_peek_prev_min(iter, POS_MIN); 416 + return bch2_btree_iter_peek_prev_min(trans, iter, POS_MIN); 416 417 } 417 418 418 - struct bkey_s_c bch2_btree_iter_prev(struct btree_iter *); 419 + struct bkey_s_c bch2_btree_iter_prev(struct btree_trans *, struct btree_iter *); 419 420 420 - struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *); 421 - struct bkey_s_c bch2_btree_iter_next_slot(struct btree_iter *); 422 - struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_iter *); 421 + struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_trans *, struct btree_iter *); 422 + struct bkey_s_c bch2_btree_iter_next_slot(struct btree_trans *, struct btree_iter *); 423 + struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_trans *, struct btree_iter *); 423 424 424 - bool bch2_btree_iter_advance(struct btree_iter *); 425 - bool bch2_btree_iter_rewind(struct btree_iter *); 425 + bool bch2_btree_iter_advance(struct btree_trans *, struct btree_iter *); 426 + bool bch2_btree_iter_rewind(struct btree_trans *, struct btree_iter *); 426 427 427 428 static inline void __bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos) 428 429 { ··· 434 433 iter->k.size = 0; 435 434 } 436 435 437 - static inline void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos) 436 + static inline void bch2_btree_iter_set_pos(struct btree_trans *trans, 437 + struct btree_iter *iter, struct bpos new_pos) 438 438 { 439 - struct btree_trans *trans = iter->trans; 440 - 441 439 if (unlikely(iter->update_path)) 442 440 bch2_path_put(trans, iter->update_path, 443 441 iter->flags & BTREE_ITER_intent); ··· 454 454 iter->pos = bkey_start_pos(&iter->k); 455 455 } 456 456 457 - static inline void bch2_btree_iter_set_snapshot(struct btree_iter *iter, u32 snapshot) 457 + static inline void bch2_btree_iter_set_snapshot(struct btree_trans *trans, 458 + struct btree_iter *iter, u32 snapshot) 458 459 { 459 460 struct bpos pos = iter->pos; 460 461 461 462 iter->snapshot = snapshot; 462 463 pos.snapshot = snapshot; 463 - bch2_btree_iter_set_pos(iter, pos); 464 + bch2_btree_iter_set_pos(trans, iter, pos); 464 465 } 465 466 466 467 void bch2_trans_iter_exit(struct btree_trans *, struct btree_iter *); ··· 503 502 unsigned flags, 504 503 unsigned long ip) 505 504 { 506 - iter->trans = trans; 507 505 iter->update_path = 0; 508 506 iter->key_cache_path = 0; 509 507 iter->btree_id = btree_id; ··· 539 539 void bch2_trans_node_iter_init(struct btree_trans *, struct btree_iter *, 540 540 enum btree_id, struct bpos, 541 541 unsigned, unsigned, unsigned); 542 - void bch2_trans_copy_iter(struct btree_iter *, struct btree_iter *); 542 + void bch2_trans_copy_iter(struct btree_trans *, struct btree_iter *, struct btree_iter *); 543 543 544 - void bch2_set_btree_iter_dontneed(struct btree_iter *); 544 + void bch2_set_btree_iter_dontneed(struct btree_trans *, struct btree_iter *); 545 545 546 546 void *__bch2_trans_kmalloc(struct btree_trans *, size_t); 547 547 ··· 588 588 struct bkey_s_c k; 589 589 590 590 bch2_trans_iter_init(trans, iter, btree_id, pos, flags); 591 - k = bch2_btree_iter_peek_slot(iter); 591 + k = bch2_btree_iter_peek_slot(trans, iter); 592 592 593 593 if (!bkey_err(k) && type && k.k->type != type) 594 594 k = bkey_s_c_err(-BCH_ERR_ENOENT_bkey_type_mismatch); ··· 658 658 int _ret3 = 0; \ 659 659 do { \ 660 660 _ret3 = lockrestart_do((_trans), ({ \ 661 - struct btree *_b = bch2_btree_iter_peek_node(&_iter); \ 661 + struct btree *_b = bch2_btree_iter_peek_node(_trans, &_iter);\ 662 662 if (!_b) \ 663 663 break; \ 664 664 \ 665 665 PTR_ERR_OR_ZERO(_b) ?: (_do); \ 666 666 })) ?: \ 667 667 lockrestart_do((_trans), \ 668 - PTR_ERR_OR_ZERO(bch2_btree_iter_next_node(&_iter))); \ 668 + PTR_ERR_OR_ZERO(bch2_btree_iter_next_node(_trans, &_iter)));\ 669 669 } while (!_ret3); \ 670 670 \ 671 671 bch2_trans_iter_exit((_trans), &(_iter)); \ ··· 677 677 __for_each_btree_node(_trans, _iter, _btree_id, _start, \ 678 678 0, 0, _flags, _b, _do) 679 679 680 - static inline struct bkey_s_c bch2_btree_iter_peek_prev_type(struct btree_iter *iter, 680 + static inline struct bkey_s_c bch2_btree_iter_peek_prev_type(struct btree_trans *trans, 681 + struct btree_iter *iter, 681 682 unsigned flags) 682 683 { 683 - return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(iter) : 684 - bch2_btree_iter_peek_prev(iter); 684 + return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(trans, iter) : 685 + bch2_btree_iter_peek_prev(trans, iter); 685 686 } 686 687 687 - static inline struct bkey_s_c bch2_btree_iter_peek_type(struct btree_iter *iter, 688 + static inline struct bkey_s_c bch2_btree_iter_peek_type(struct btree_trans *trans, 689 + struct btree_iter *iter, 688 690 unsigned flags) 689 691 { 690 - return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(iter) : 691 - bch2_btree_iter_peek(iter); 692 + return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(trans, iter) : 693 + bch2_btree_iter_peek(trans, iter); 692 694 } 693 695 694 - static inline struct bkey_s_c bch2_btree_iter_peek_max_type(struct btree_iter *iter, 695 - struct bpos end, 696 - unsigned flags) 696 + static inline struct bkey_s_c bch2_btree_iter_peek_max_type(struct btree_trans *trans, 697 + struct btree_iter *iter, 698 + struct bpos end, 699 + unsigned flags) 697 700 { 698 701 if (!(flags & BTREE_ITER_slots)) 699 - return bch2_btree_iter_peek_max(iter, end); 702 + return bch2_btree_iter_peek_max(trans, iter, end); 700 703 701 704 if (bkey_gt(iter->pos, end)) 702 705 return bkey_s_c_null; 703 706 704 - return bch2_btree_iter_peek_slot(iter); 707 + return bch2_btree_iter_peek_slot(trans, iter); 705 708 } 706 709 707 710 int __bch2_btree_trans_too_many_iters(struct btree_trans *); ··· 771 768 \ 772 769 do { \ 773 770 _ret3 = lockrestart_do(_trans, ({ \ 774 - (_k) = bch2_btree_iter_peek_max_type(&(_iter), \ 771 + (_k) = bch2_btree_iter_peek_max_type(_trans, &(_iter), \ 775 772 _end, (_flags)); \ 776 773 if (!(_k).k) \ 777 774 break; \ 778 775 \ 779 776 bkey_err(_k) ?: (_do); \ 780 777 })); \ 781 - } while (!_ret3 && bch2_btree_iter_advance(&(_iter))); \ 778 + } while (!_ret3 && bch2_btree_iter_advance(_trans, &(_iter))); \ 782 779 \ 783 780 bch2_trans_iter_exit((_trans), &(_iter)); \ 784 781 _ret3; \ ··· 816 813 \ 817 814 do { \ 818 815 _ret3 = lockrestart_do(_trans, ({ \ 819 - (_k) = bch2_btree_iter_peek_prev_type(&(_iter), \ 816 + (_k) = bch2_btree_iter_peek_prev_type(_trans, &(_iter), \ 820 817 (_flags)); \ 821 818 if (!(_k).k) \ 822 819 break; \ 823 820 \ 824 821 bkey_err(_k) ?: (_do); \ 825 822 })); \ 826 - } while (!_ret3 && bch2_btree_iter_rewind(&(_iter))); \ 823 + } while (!_ret3 && bch2_btree_iter_rewind(_trans, &(_iter))); \ 827 824 \ 828 825 bch2_trans_iter_exit((_trans), &(_iter)); \ 829 826 _ret3; \ ··· 853 850 (_do) ?: bch2_trans_commit(_trans, (_disk_res),\ 854 851 (_journal_seq), (_commit_flags))) 855 852 856 - struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *); 853 + struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_trans *, 854 + struct btree_iter *); 857 855 858 856 #define for_each_btree_key_max_norestart(_trans, _iter, _btree_id, \ 859 857 _start, _end, _flags, _k, _ret) \ 860 858 for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ 861 859 (_start), (_flags)); \ 862 - (_k) = bch2_btree_iter_peek_max_type(&(_iter), _end, _flags),\ 860 + (_k) = bch2_btree_iter_peek_max_type(_trans, &(_iter), _end, _flags),\ 863 861 !((_ret) = bkey_err(_k)) && (_k).k; \ 864 - bch2_btree_iter_advance(&(_iter))) 862 + bch2_btree_iter_advance(_trans, &(_iter))) 865 863 866 - #define for_each_btree_key_max_continue_norestart(_iter, _end, _flags, _k, _ret)\ 864 + #define for_each_btree_key_max_continue_norestart(_trans, _iter, _end, _flags, _k, _ret)\ 867 865 for (; \ 868 - (_k) = bch2_btree_iter_peek_max_type(&(_iter), _end, _flags), \ 866 + (_k) = bch2_btree_iter_peek_max_type(_trans, &(_iter), _end, _flags), \ 869 867 !((_ret) = bkey_err(_k)) && (_k).k; \ 870 - bch2_btree_iter_advance(&(_iter))) 868 + bch2_btree_iter_advance(_trans, &(_iter))) 871 869 872 870 #define for_each_btree_key_norestart(_trans, _iter, _btree_id, \ 873 871 _start, _flags, _k, _ret) \ 874 872 for_each_btree_key_max_norestart(_trans, _iter, _btree_id, _start,\ 875 873 SPOS_MAX, _flags, _k, _ret) 876 874 877 - #define for_each_btree_key_reverse_norestart(_trans, _iter, _btree_id, \ 878 - _start, _flags, _k, _ret) \ 879 - for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ 880 - (_start), (_flags)); \ 881 - (_k) = bch2_btree_iter_peek_prev_type(&(_iter), _flags), \ 882 - !((_ret) = bkey_err(_k)) && (_k).k; \ 883 - bch2_btree_iter_rewind(&(_iter))) 875 + #define for_each_btree_key_reverse_norestart(_trans, _iter, _btree_id, \ 876 + _start, _flags, _k, _ret) \ 877 + for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ 878 + (_start), (_flags)); \ 879 + (_k) = bch2_btree_iter_peek_prev_type(_trans, &(_iter), _flags), \ 880 + !((_ret) = bkey_err(_k)) && (_k).k; \ 881 + bch2_btree_iter_rewind(_trans, &(_iter))) 884 882 885 - #define for_each_btree_key_continue_norestart(_iter, _flags, _k, _ret) \ 886 - for_each_btree_key_max_continue_norestart(_iter, SPOS_MAX, _flags, _k, _ret) 883 + #define for_each_btree_key_continue_norestart(_trans, _iter, _flags, _k, _ret) \ 884 + for_each_btree_key_max_continue_norestart(_trans, _iter, SPOS_MAX, _flags, _k, _ret) 887 885 888 886 /* 889 887 * This should not be used in a fastpath, without first trying _do in

+19 -13

fs/bcachefs/btree_key_cache.c

··· 287 287 return ret; 288 288 } 289 289 290 + static noinline_for_stack void do_trace_key_cache_fill(struct btree_trans *trans, 291 + struct btree_path *ck_path, 292 + struct bkey_s_c k) 293 + { 294 + struct printbuf buf = PRINTBUF; 295 + 296 + bch2_bpos_to_text(&buf, ck_path->pos); 297 + prt_char(&buf, ' '); 298 + bch2_bkey_val_to_text(&buf, trans->c, k); 299 + trace_key_cache_fill(trans, buf.buf); 300 + printbuf_exit(&buf); 301 + } 302 + 290 303 static noinline int btree_key_cache_fill(struct btree_trans *trans, 291 304 struct btree_path *ck_path, 292 305 unsigned flags) ··· 319 306 BTREE_ITER_key_cache_fill| 320 307 BTREE_ITER_cached_nofill); 321 308 iter.flags &= ~BTREE_ITER_with_journal; 322 - k = bch2_btree_iter_peek_slot(&iter); 309 + k = bch2_btree_iter_peek_slot(trans, &iter); 323 310 ret = bkey_err(k); 324 311 if (ret) 325 312 goto err; ··· 333 320 if (ret) 334 321 goto err; 335 322 336 - if (trace_key_cache_fill_enabled()) { 337 - struct printbuf buf = PRINTBUF; 338 - 339 - bch2_bpos_to_text(&buf, ck_path->pos); 340 - prt_char(&buf, ' '); 341 - bch2_bkey_val_to_text(&buf, trans->c, k); 342 - trace_key_cache_fill(trans, buf.buf); 343 - printbuf_exit(&buf); 344 - } 323 + if (trace_key_cache_fill_enabled()) 324 + do_trace_key_cache_fill(trans, ck_path, k); 345 325 out: 346 326 /* We're not likely to need this iterator again: */ 347 - bch2_set_btree_iter_dontneed(&iter); 327 + bch2_set_btree_iter_dontneed(trans, &iter); 348 328 err: 349 329 bch2_trans_iter_exit(trans, &iter); 350 330 return ret; ··· 418 412 BTREE_ITER_intent); 419 413 b_iter.flags &= ~BTREE_ITER_with_key_cache; 420 414 421 - ret = bch2_btree_iter_traverse(&c_iter); 415 + ret = bch2_btree_iter_traverse(trans, &c_iter); 422 416 if (ret) 423 417 goto out; 424 418 ··· 450 444 !test_bit(JOURNAL_space_low, &c->journal.flags)) 451 445 commit_flags |= BCH_TRANS_COMMIT_no_journal_res; 452 446 453 - struct bkey_s_c btree_k = bch2_btree_iter_peek_slot(&b_iter); 447 + struct bkey_s_c btree_k = bch2_btree_iter_peek_slot(trans, &b_iter); 454 448 ret = bkey_err(btree_k); 455 449 if (ret) 456 450 goto err;

+4 -4

fs/bcachefs/btree_node_scan.c

··· 271 271 err: 272 272 bio_put(bio); 273 273 free_page((unsigned long) buf); 274 - percpu_ref_put(&ca->io_ref); 274 + percpu_ref_put(&ca->io_ref[READ]); 275 275 closure_put(w->cl); 276 276 kfree(w); 277 277 return 0; ··· 291 291 292 292 struct find_btree_nodes_worker *w = kmalloc(sizeof(*w), GFP_KERNEL); 293 293 if (!w) { 294 - percpu_ref_put(&ca->io_ref); 294 + percpu_ref_put(&ca->io_ref[READ]); 295 295 ret = -ENOMEM; 296 296 goto err; 297 297 } ··· 303 303 struct task_struct *t = kthread_create(read_btree_nodes_worker, w, "read_btree_nodes/%s", ca->name); 304 304 ret = PTR_ERR_OR_ZERO(t); 305 305 if (ret) { 306 - percpu_ref_put(&ca->io_ref); 306 + percpu_ref_put(&ca->io_ref[READ]); 307 307 kfree(w); 308 308 bch_err_msg(c, ret, "starting kthread"); 309 309 break; 310 310 } 311 311 312 312 closure_get(&cl); 313 - percpu_ref_get(&ca->io_ref); 313 + percpu_ref_get(&ca->io_ref[READ]); 314 314 wake_up_process(t); 315 315 } 316 316 err:

-1

fs/bcachefs/btree_types.h

··· 367 367 * @nodes_intent_locked - bitmask indicating which locks are intent locks 368 368 */ 369 369 struct btree_iter { 370 - struct btree_trans *trans; 371 370 btree_path_idx_t path; 372 371 btree_path_idx_t update_path; 373 372 btree_path_idx_t key_cache_path;

+13 -13

fs/bcachefs/btree_update.c

··· 126 126 struct bpos new_pos) 127 127 { 128 128 struct bch_fs *c = trans->c; 129 - struct btree_iter old_iter, new_iter = { NULL }; 129 + struct btree_iter old_iter, new_iter = {}; 130 130 struct bkey_s_c old_k, new_k; 131 131 snapshot_id_list s; 132 132 struct bkey_i *update; ··· 140 140 bch2_trans_iter_init(trans, &old_iter, id, old_pos, 141 141 BTREE_ITER_not_extents| 142 142 BTREE_ITER_all_snapshots); 143 - while ((old_k = bch2_btree_iter_prev(&old_iter)).k && 143 + while ((old_k = bch2_btree_iter_prev(trans, &old_iter)).k && 144 144 !(ret = bkey_err(old_k)) && 145 145 bkey_eq(old_pos, old_k.k->p)) { 146 146 struct bpos whiteout_pos = ··· 296 296 BTREE_ITER_intent| 297 297 BTREE_ITER_with_updates| 298 298 BTREE_ITER_not_extents); 299 - k = bch2_btree_iter_peek_max(&iter, POS(insert->k.p.inode, U64_MAX)); 299 + k = bch2_btree_iter_peek_max(trans, &iter, POS(insert->k.p.inode, U64_MAX)); 300 300 if ((ret = bkey_err(k))) 301 301 goto err; 302 302 if (!k.k) ··· 322 322 if (done) 323 323 goto out; 324 324 next: 325 - bch2_btree_iter_advance(&iter); 326 - k = bch2_btree_iter_peek_max(&iter, POS(insert->k.p.inode, U64_MAX)); 325 + bch2_btree_iter_advance(trans, &iter); 326 + k = bch2_btree_iter_peek_max(trans, &iter, POS(insert->k.p.inode, U64_MAX)); 327 327 if ((ret = bkey_err(k))) 328 328 goto err; 329 329 if (!k.k) ··· 592 592 enum btree_id btree, struct bpos end) 593 593 { 594 594 bch2_trans_iter_init(trans, iter, btree, end, BTREE_ITER_intent); 595 - struct bkey_s_c k = bch2_btree_iter_peek_prev(iter); 595 + struct bkey_s_c k = bch2_btree_iter_peek_prev(trans, iter); 596 596 int ret = bkey_err(k); 597 597 if (ret) 598 598 goto err; 599 599 600 - bch2_btree_iter_advance(iter); 601 - k = bch2_btree_iter_peek_slot(iter); 600 + bch2_btree_iter_advance(trans, iter); 601 + k = bch2_btree_iter_peek_slot(trans, iter); 602 602 ret = bkey_err(k); 603 603 if (ret) 604 604 goto err; ··· 634 634 BTREE_ITER_cached| 635 635 BTREE_ITER_not_extents| 636 636 BTREE_ITER_intent); 637 - ret = bch2_btree_iter_traverse(&iter) ?: 637 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 638 638 bch2_trans_update(trans, &iter, k, flags); 639 639 bch2_trans_iter_exit(trans, &iter); 640 640 return ret; ··· 646 646 struct btree_iter iter; 647 647 bch2_trans_iter_init(trans, &iter, id, bkey_start_pos(&k->k), 648 648 BTREE_ITER_intent|flags); 649 - int ret = bch2_btree_iter_traverse(&iter) ?: 649 + int ret = bch2_btree_iter_traverse(trans, &iter) ?: 650 650 bch2_trans_update(trans, &iter, k, flags); 651 651 bch2_trans_iter_exit(trans, &iter); 652 652 return ret; ··· 695 695 bch2_trans_iter_init(trans, &iter, btree, pos, 696 696 BTREE_ITER_cached| 697 697 BTREE_ITER_intent); 698 - ret = bch2_btree_iter_traverse(&iter) ?: 698 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 699 699 bch2_btree_delete_at(trans, &iter, update_flags); 700 700 bch2_trans_iter_exit(trans, &iter); 701 701 ··· 713 713 int ret = 0; 714 714 715 715 bch2_trans_iter_init(trans, &iter, id, start, BTREE_ITER_intent); 716 - while ((k = bch2_btree_iter_peek_max(&iter, end)).k) { 716 + while ((k = bch2_btree_iter_peek_max(trans, &iter, end)).k) { 717 717 struct disk_reservation disk_res = 718 718 bch2_disk_reservation_init(trans->c, 0); 719 719 struct bkey_i delete; ··· 808 808 struct btree_iter iter; 809 809 bch2_trans_iter_init(trans, &iter, btree, pos, BTREE_ITER_intent); 810 810 811 - int ret = bch2_btree_iter_traverse(&iter) ?: 811 + int ret = bch2_btree_iter_traverse(trans, &iter) ?: 812 812 bch2_btree_bit_mod_iter(trans, &iter, set); 813 813 bch2_trans_iter_exit(trans, &iter); 814 814 return ret;

+6 -6

fs/bcachefs/btree_update_interior.c

··· 2147 2147 bch2_trans_node_iter_init(trans, iter, b->c.btree_id, b->key.k.p, 2148 2148 BTREE_MAX_DEPTH, b->c.level, 2149 2149 BTREE_ITER_intent); 2150 - int ret = bch2_btree_iter_traverse(iter); 2150 + int ret = bch2_btree_iter_traverse(trans, iter); 2151 2151 if (ret) 2152 2152 goto err; 2153 2153 ··· 2239 2239 bch2_trans_node_iter_init(trans, &iter, 2240 2240 btree, k->k.p, 2241 2241 BTREE_MAX_DEPTH, level, 0); 2242 - struct btree *b = bch2_btree_iter_peek_node(&iter); 2242 + struct btree *b = bch2_btree_iter_peek_node(trans, &iter); 2243 2243 int ret = PTR_ERR_OR_ZERO(b); 2244 2244 if (ret) 2245 2245 goto out; ··· 2262 2262 /* Traverse one depth lower to get a pointer to the node itself: */ 2263 2263 struct btree_iter iter; 2264 2264 bch2_trans_node_iter_init(trans, &iter, btree, pos, 0, level - 1, 0); 2265 - struct btree *b = bch2_btree_iter_peek_node(&iter); 2265 + struct btree *b = bch2_btree_iter_peek_node(trans, &iter); 2266 2266 int ret = PTR_ERR_OR_ZERO(b); 2267 2267 if (ret) 2268 2268 goto err; ··· 2406 2406 bool skip_triggers) 2407 2407 { 2408 2408 struct bch_fs *c = trans->c; 2409 - struct btree_iter iter2 = { NULL }; 2409 + struct btree_iter iter2 = {}; 2410 2410 struct btree *parent; 2411 2411 int ret; 2412 2412 ··· 2430 2430 2431 2431 parent = btree_node_parent(btree_iter_path(trans, iter), b); 2432 2432 if (parent) { 2433 - bch2_trans_copy_iter(&iter2, iter); 2433 + bch2_trans_copy_iter(trans, &iter2, iter); 2434 2434 2435 2435 iter2.path = bch2_btree_path_make_mut(trans, iter2.path, 2436 2436 iter2.flags & BTREE_ITER_intent, ··· 2444 2444 2445 2445 trans->paths_sorted = false; 2446 2446 2447 - ret = bch2_btree_iter_traverse(&iter2) ?: 2447 + ret = bch2_btree_iter_traverse(trans, &iter2) ?: 2448 2448 bch2_trans_update(trans, &iter2, new_key, BTREE_TRIGGER_norun); 2449 2449 if (ret) 2450 2450 goto err;

+5 -5

fs/bcachefs/btree_write_buffer.c

··· 144 144 EBUG_ON(!trans->c->btree_write_buffer.flushing.pin.seq); 145 145 EBUG_ON(trans->c->btree_write_buffer.flushing.pin.seq > wb->journal_seq); 146 146 147 - ret = bch2_btree_iter_traverse(iter); 147 + ret = bch2_btree_iter_traverse(trans, iter); 148 148 if (ret) 149 149 return ret; 150 150 ··· 208 208 209 209 trans->journal_res.seq = wb->journal_seq; 210 210 211 - ret = bch2_btree_iter_traverse(&iter) ?: 211 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 212 212 bch2_trans_update(trans, &iter, &wb->k, 213 213 BTREE_UPDATE_internal_snapshot_node); 214 214 bch2_trans_iter_exit(trans, &iter); ··· 285 285 struct bch_fs *c = trans->c; 286 286 struct journal *j = &c->journal; 287 287 struct btree_write_buffer *wb = &c->btree_write_buffer; 288 - struct btree_iter iter = { NULL }; 288 + struct btree_iter iter = {}; 289 289 size_t overwritten = 0, fast = 0, slowpath = 0, could_not_insert = 0; 290 290 bool write_locked = false; 291 291 bool accounting_replay_done = test_bit(BCH_FS_accounting_replay_done, &c->flags); ··· 368 368 write_locked = false; 369 369 370 370 ret = lockrestart_do(trans, 371 - bch2_btree_iter_traverse(&iter) ?: 371 + bch2_btree_iter_traverse(trans, &iter) ?: 372 372 bch2_foreground_maybe_merge(trans, iter.path, 0, 373 373 BCH_WATERMARK_reclaim| 374 374 BCH_TRANS_COMMIT_journal_reclaim| ··· 385 385 BTREE_ITER_intent|BTREE_ITER_all_snapshots); 386 386 } 387 387 388 - bch2_btree_iter_set_pos(&iter, k->k.k.p); 388 + bch2_btree_iter_set_pos(trans, &iter, k->k.k.p); 389 389 btree_iter_path(trans, &iter)->preserve = false; 390 390 391 391 bool accounting_accumulated = false;

+11 -5

fs/bcachefs/buckets.c

··· 30 30 31 31 void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage) 32 32 { 33 + for (unsigned i = 0; i < BCH_DATA_NR; i++) 34 + usage->buckets[i] = percpu_u64_get(&ca->usage->d[i].buckets); 35 + } 36 + 37 + void bch2_dev_usage_full_read_fast(struct bch_dev *ca, struct bch_dev_usage_full *usage) 38 + { 33 39 memset(usage, 0, sizeof(*usage)); 34 40 acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage, dev_usage_u64s()); 35 41 } ··· 81 75 82 76 void bch2_dev_usage_to_text(struct printbuf *out, 83 77 struct bch_dev *ca, 84 - struct bch_dev_usage *usage) 78 + struct bch_dev_usage_full *usage) 85 79 { 86 80 if (out->nr_tabstops < 5) { 87 81 printbuf_tabstops_reset(out); ··· 371 365 struct btree_iter iter; 372 366 bch2_trans_node_iter_init(trans, &iter, btree, new->k.p, 0, level, 373 367 BTREE_ITER_intent|BTREE_ITER_all_snapshots); 374 - ret = bch2_btree_iter_traverse(&iter) ?: 368 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 375 369 bch2_trans_update(trans, &iter, new, 376 370 BTREE_UPDATE_internal_snapshot_node| 377 371 BTREE_TRIGGER_norun); ··· 713 707 struct disk_accounting_pos acc; 714 708 memset(&acc, 0, sizeof(acc)); 715 709 acc.type = BCH_DISK_ACCOUNTING_replicas; 716 - memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e)); 710 + unsafe_memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e), "VLA"); 717 711 gc_stripe_unlock(m); 718 712 719 713 acc.replicas.data_type = data_type; ··· 1138 1132 for_each_online_member(c, ca) { 1139 1133 int ret = bch2_trans_mark_dev_sb(c, ca, flags); 1140 1134 if (ret) { 1141 - percpu_ref_put(&ca->io_ref); 1135 + percpu_ref_put(&ca->io_ref[READ]); 1142 1136 return ret; 1143 1137 } 1144 1138 } ··· 1337 1331 1338 1332 int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca) 1339 1333 { 1340 - ca->usage = alloc_percpu(struct bch_dev_usage); 1334 + ca->usage = alloc_percpu(struct bch_dev_usage_full); 1341 1335 if (!ca->usage) 1342 1336 return -BCH_ERR_ENOMEM_usage_init; 1343 1337

+15 -6

fs/bcachefs/buckets.h

··· 172 172 return ret; 173 173 } 174 174 175 - void bch2_dev_usage_to_text(struct printbuf *, struct bch_dev *, struct bch_dev_usage *); 175 + void bch2_dev_usage_full_read_fast(struct bch_dev *, struct bch_dev_usage_full *); 176 + static inline struct bch_dev_usage_full bch2_dev_usage_full_read(struct bch_dev *ca) 177 + { 178 + struct bch_dev_usage_full ret; 179 + 180 + bch2_dev_usage_full_read_fast(ca, &ret); 181 + return ret; 182 + } 183 + 184 + void bch2_dev_usage_to_text(struct printbuf *, struct bch_dev *, struct bch_dev_usage_full *); 176 185 177 186 static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_watermark watermark) 178 187 { ··· 216 207 enum bch_watermark watermark) 217 208 { 218 209 return max_t(s64, 0, 219 - usage.d[BCH_DATA_free].buckets - 210 + usage.buckets[BCH_DATA_free]- 220 211 ca->nr_open_buckets - 221 212 bch2_dev_buckets_reserved(ca, watermark)); 222 213 } ··· 226 217 enum bch_watermark watermark) 227 218 { 228 219 return max_t(s64, 0, 229 - usage.d[BCH_DATA_free].buckets 230 - + usage.d[BCH_DATA_cached].buckets 231 - + usage.d[BCH_DATA_need_gc_gens].buckets 232 - + usage.d[BCH_DATA_need_discard].buckets 220 + usage.buckets[BCH_DATA_free] 221 + + usage.buckets[BCH_DATA_cached] 222 + + usage.buckets[BCH_DATA_need_gc_gens] 223 + + usage.buckets[BCH_DATA_need_discard] 233 224 - ca->nr_open_buckets 234 225 - bch2_dev_buckets_reserved(ca, watermark)); 235 226 }

+5

fs/bcachefs/buckets_types.h

··· 54 54 u8 b[] __counted_by(nbuckets); 55 55 }; 56 56 57 + /* Only info on bucket countns: */ 57 58 struct bch_dev_usage { 59 + u64 buckets[BCH_DATA_NR]; 60 + }; 61 + 62 + struct bch_dev_usage_full { 58 63 struct bch_dev_usage_type { 59 64 u64 buckets; 60 65 u64 sectors; /* _compressed_ sectors: */

+7 -7

fs/bcachefs/chardev.c

··· 350 350 if (ctx->arg.op == BCH_DATA_OP_scrub) { 351 351 struct bch_dev *ca = bch2_dev_tryget(c, ctx->arg.scrub.dev); 352 352 if (ca) { 353 - struct bch_dev_usage u; 354 - bch2_dev_usage_read_fast(ca, &u); 353 + struct bch_dev_usage_full u; 354 + bch2_dev_usage_full_read_fast(ca, &u); 355 355 for (unsigned i = BCH_DATA_btree; i < ARRAY_SIZE(u.d); i++) 356 356 if (ctx->arg.scrub.data_types & BIT(i)) 357 357 e.p.sectors_total += u.d[i].sectors; ··· 473 473 struct bch_ioctl_dev_usage __user *user_arg) 474 474 { 475 475 struct bch_ioctl_dev_usage arg; 476 - struct bch_dev_usage src; 476 + struct bch_dev_usage_full src; 477 477 struct bch_dev *ca; 478 478 unsigned i; 479 479 ··· 493 493 if (IS_ERR(ca)) 494 494 return PTR_ERR(ca); 495 495 496 - src = bch2_dev_usage_read(ca); 496 + src = bch2_dev_usage_full_read(ca); 497 497 498 498 arg.state = ca->mi.state; 499 499 arg.bucket_size = ca->mi.bucket_size; ··· 514 514 struct bch_ioctl_dev_usage_v2 __user *user_arg) 515 515 { 516 516 struct bch_ioctl_dev_usage_v2 arg; 517 - struct bch_dev_usage src; 517 + struct bch_dev_usage_full src; 518 518 struct bch_dev *ca; 519 519 int ret = 0; 520 520 ··· 534 534 if (IS_ERR(ca)) 535 535 return PTR_ERR(ca); 536 536 537 - src = bch2_dev_usage_read(ca); 537 + src = bch2_dev_usage_full_read(ca); 538 538 539 539 arg.state = ca->mi.state; 540 540 arg.bucket_size = ca->mi.bucket_size; ··· 615 615 616 616 for_each_online_member(c, ca) 617 617 if (ca->dev == dev) { 618 - percpu_ref_put(&ca->io_ref); 618 + percpu_ref_put(&ca->io_ref[READ]); 619 619 return ca->dev_idx; 620 620 } 621 621

+3 -2

fs/bcachefs/compress.c

··· 371 371 }; 372 372 373 373 zlib_set_workspace(&strm, workspace); 374 - zlib_deflateInit2(&strm, 374 + if (zlib_deflateInit2(&strm, 375 375 compression.level 376 376 ? clamp_t(unsigned, compression.level, 377 377 Z_BEST_SPEED, Z_BEST_COMPRESSION) 378 378 : Z_DEFAULT_COMPRESSION, 379 379 Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, 380 - Z_DEFAULT_STRATEGY); 380 + Z_DEFAULT_STRATEGY) != Z_OK) 381 + return 0; 381 382 382 383 if (zlib_deflate(&strm, Z_FINISH) != Z_STREAM_END) 383 384 return 0;

+4 -4

fs/bcachefs/data_update.c

··· 216 216 217 217 bch2_trans_begin(trans); 218 218 219 - k = bch2_btree_iter_peek_slot(&iter); 219 + k = bch2_btree_iter_peek_slot(trans, &iter); 220 220 ret = bkey_err(k); 221 221 if (ret) 222 222 goto err; ··· 398 398 BCH_TRANS_COMMIT_no_enospc| 399 399 m->data_opts.btree_insert_flags); 400 400 if (!ret) { 401 - bch2_btree_iter_set_pos(&iter, next_pos); 401 + bch2_btree_iter_set_pos(trans, &iter, next_pos); 402 402 403 403 this_cpu_add(c->counters[BCH_COUNTER_io_move_finish], new->k.size); 404 404 if (trace_io_move_finish_enabled()) ··· 426 426 427 427 count_event(c, io_move_fail); 428 428 429 - bch2_btree_iter_advance(&iter); 429 + bch2_btree_iter_advance(trans, &iter); 430 430 goto next; 431 431 } 432 432 out: ··· 497 497 bch2_trans_iter_init(trans, &iter, update->btree_id, update->op.pos, 498 498 BTREE_ITER_slots); 499 499 ret = lockrestart_do(trans, ({ 500 - k = bch2_btree_iter_peek_slot(&iter); 500 + k = bch2_btree_iter_peek_slot(trans, &iter); 501 501 bkey_err(k); 502 502 })); 503 503 bch2_trans_iter_exit(trans, &iter);

+2 -2

fs/bcachefs/debug.c

··· 57 57 submit_bio_wait(bio); 58 58 59 59 bio_put(bio); 60 - percpu_ref_put(&ca->io_ref); 60 + percpu_ref_put(&ca->io_ref[READ]); 61 61 62 62 memcpy(n_ondisk, n_sorted, btree_buf_bytes(b)); 63 63 ··· 297 297 if (bio) 298 298 bio_put(bio); 299 299 kvfree(n_ondisk); 300 - percpu_ref_put(&ca->io_ref); 300 + percpu_ref_put(&ca->io_ref[READ]); 301 301 } 302 302 303 303 #ifdef CONFIG_DEBUG_FS

+8 -8

fs/bcachefs/dirent.c

··· 417 417 enum bch_rename_mode mode) 418 418 { 419 419 struct qstr src_name_lookup, dst_name_lookup; 420 - struct btree_iter src_iter = { NULL }; 421 - struct btree_iter dst_iter = { NULL }; 420 + struct btree_iter src_iter = {}; 421 + struct btree_iter dst_iter = {}; 422 422 struct bkey_s_c old_src, old_dst = bkey_s_c_null; 423 423 struct bkey_i_dirent *new_src = NULL, *new_dst = NULL; 424 424 struct bpos dst_pos = ··· 586 586 } 587 587 588 588 if (delete_src) { 589 - bch2_btree_iter_set_snapshot(&src_iter, old_src.k->p.snapshot); 590 - ret = bch2_btree_iter_traverse(&src_iter) ?: 589 + bch2_btree_iter_set_snapshot(trans, &src_iter, old_src.k->p.snapshot); 590 + ret = bch2_btree_iter_traverse(trans, &src_iter) ?: 591 591 bch2_btree_delete_at(trans, &src_iter, BTREE_UPDATE_internal_snapshot_node); 592 592 if (ret) 593 593 goto out; 594 594 } 595 595 596 596 if (delete_dst) { 597 - bch2_btree_iter_set_snapshot(&dst_iter, old_dst.k->p.snapshot); 598 - ret = bch2_btree_iter_traverse(&dst_iter) ?: 597 + bch2_btree_iter_set_snapshot(trans, &dst_iter, old_dst.k->p.snapshot); 598 + ret = bch2_btree_iter_traverse(trans, &dst_iter) ?: 599 599 bch2_btree_delete_at(trans, &dst_iter, BTREE_UPDATE_internal_snapshot_node); 600 600 if (ret) 601 601 goto out; ··· 642 642 const struct qstr *name, subvol_inum *inum) 643 643 { 644 644 struct btree_trans *trans = bch2_trans_get(c); 645 - struct btree_iter iter = { NULL }; 645 + struct btree_iter iter = {}; 646 646 647 647 int ret = lockrestart_do(trans, 648 648 bch2_dirent_lookup_trans(trans, &iter, dir, hash_info, name, inum, 0)); ··· 771 771 772 772 bch2_trans_iter_init(trans, &iter, BTREE_ID_dirents, pos, BTREE_ITER_intent); 773 773 774 - ret = bch2_btree_iter_traverse(&iter) ?: 774 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 775 775 bch2_hash_delete_at(trans, bch2_dirent_hash_desc, 776 776 &dir_hash_info, &iter, 777 777 BTREE_UPDATE_internal_snapshot_node);

+2 -2

fs/bcachefs/disk_accounting.c

··· 739 739 struct disk_accounting_pos next; 740 740 memset(&next, 0, sizeof(next)); 741 741 next.type = acc_k.type + 1; 742 - bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next)); 742 + bch2_btree_iter_set_pos(trans, &iter, disk_accounting_pos_to_bpos(&next)); 743 743 continue; 744 744 } 745 745 ··· 930 930 struct disk_accounting_pos next; 931 931 memset(&next, 0, sizeof(next)); 932 932 next.type = acc_k.type + 1; 933 - bch2_btree_iter_set_pos(&iter, disk_accounting_pos_to_bpos(&next)); 933 + bch2_btree_iter_set_pos(trans, &iter, disk_accounting_pos_to_bpos(&next)); 934 934 continue; 935 935 } 936 936

+2 -2

fs/bcachefs/disk_groups.c

··· 555 555 ? rcu_dereference(c->devs[t.dev]) 556 556 : NULL; 557 557 558 - if (ca && percpu_ref_tryget(&ca->io_ref)) { 558 + if (ca && percpu_ref_tryget(&ca->io_ref[READ])) { 559 559 prt_printf(out, "/dev/%s", ca->name); 560 - percpu_ref_put(&ca->io_ref); 560 + percpu_ref_put(&ca->io_ref[READ]); 561 561 } else if (ca) { 562 562 prt_printf(out, "offline device %u", t.dev); 563 563 } else {

+11 -7

fs/bcachefs/ec.c

··· 105 105 struct bch_dev *ca; 106 106 struct ec_stripe_buf *buf; 107 107 size_t idx; 108 + int rw; 108 109 u64 submit_time; 109 110 struct bio bio; 110 111 }; ··· 463 462 return ret; 464 463 465 464 if (gc) 466 - memcpy(&gc->r.e, &acc.replicas, replicas_entry_bytes(&acc.replicas)); 465 + unsafe_memcpy(&gc->r.e, &acc.replicas, 466 + replicas_entry_bytes(&acc.replicas), "VLA"); 467 467 } 468 468 469 469 if (old_s) { ··· 705 703 struct bch_extent_ptr *ptr = &v->ptrs[ec_bio->idx]; 706 704 struct bch_dev *ca = ec_bio->ca; 707 705 struct closure *cl = bio->bi_private; 706 + int rw = ec_bio->rw; 708 707 709 708 bch2_account_io_completion(ca, bio_data_dir(bio), 710 709 ec_bio->submit_time, !bio->bi_status); ··· 727 724 } 728 725 729 726 bio_put(&ec_bio->bio); 730 - percpu_ref_put(&ca->io_ref); 727 + percpu_ref_put(&ca->io_ref[rw]); 731 728 closure_put(cl); 732 729 } 733 730 ··· 778 775 ec_bio->ca = ca; 779 776 ec_bio->buf = buf; 780 777 ec_bio->idx = idx; 778 + ec_bio->rw = rw; 781 779 ec_bio->submit_time = local_clock(); 782 780 783 781 ec_bio->bio.bi_iter.bi_sector = ptr->offset + buf->offset + (offset >> 9); ··· 788 784 bch2_bio_map(&ec_bio->bio, buf->data[idx] + offset, b); 789 785 790 786 closure_get(cl); 791 - percpu_ref_get(&ca->io_ref); 787 + percpu_ref_get(&ca->io_ref[rw]); 792 788 793 789 submit_bio(&ec_bio->bio); 794 790 795 791 offset += b; 796 792 } 797 793 798 - percpu_ref_put(&ca->io_ref); 794 + percpu_ref_put(&ca->io_ref[rw]); 799 795 } 800 796 801 797 static int get_stripe_key_trans(struct btree_trans *trans, u64 idx, ··· 1268 1264 ob->sectors_free, 1269 1265 GFP_KERNEL, 0); 1270 1266 1271 - percpu_ref_put(&ca->io_ref); 1267 + percpu_ref_put(&ca->io_ref[WRITE]); 1272 1268 1273 1269 if (ret) 1274 1270 s->err = ret; ··· 1840 1836 ret = 1; 1841 1837 } 1842 1838 out: 1843 - bch2_set_btree_iter_dontneed(&iter); 1839 + bch2_set_btree_iter_dontneed(trans, &iter); 1844 1840 err: 1845 1841 bch2_trans_iter_exit(trans, &iter); 1846 1842 return ret; ··· 1953 1949 if (bkey_gt(k.k->p, POS(0, U32_MAX))) { 1954 1950 if (start_pos.offset) { 1955 1951 start_pos = min_pos; 1956 - bch2_btree_iter_set_pos(&iter, start_pos); 1952 + bch2_btree_iter_set_pos(trans, &iter, start_pos); 1957 1953 continue; 1958 1954 } 1959 1955

+5 -2

fs/bcachefs/error.c

··· 34 34 journal_cur_seq(&c->journal)); 35 35 return true; 36 36 case BCH_ON_ERROR_panic: 37 - bch2_print_string_as_lines(KERN_ERR, out->buf); 37 + bch2_print_string_as_lines_nonblocking(KERN_ERR, out->buf); 38 38 panic(bch2_fmt(c, "panic after error")); 39 39 return true; 40 40 default: ··· 45 45 bool bch2_inconsistent_error(struct bch_fs *c) 46 46 { 47 47 struct printbuf buf = PRINTBUF; 48 + buf.atomic++; 49 + 48 50 printbuf_indent_add_nextline(&buf, 2); 49 51 50 52 bool ret = __bch2_inconsistent_error(c, &buf); ··· 61 59 const char *fmt, va_list args) 62 60 { 63 61 struct printbuf buf = PRINTBUF; 62 + buf.atomic++; 64 63 65 64 bch2_log_msg_start(c, &buf); 66 65 ··· 71 68 if (trans) 72 69 bch2_trans_updates_to_text(&buf, trans); 73 70 bool ret = __bch2_inconsistent_error(c, &buf); 74 - bch2_print_string_as_lines(KERN_ERR, buf.buf); 71 + bch2_print_string_as_lines_nonblocking(KERN_ERR, buf.buf); 75 72 76 73 printbuf_exit(&buf); 77 74 return ret;

+3 -3

fs/bcachefs/extent_update.c

··· 112 112 unsigned nr_iters = 0; 113 113 int ret; 114 114 115 - ret = bch2_btree_iter_traverse(iter); 115 + ret = bch2_btree_iter_traverse(trans, iter); 116 116 if (ret) 117 117 return ret; 118 118 ··· 126 126 if (ret < 0) 127 127 return ret; 128 128 129 - bch2_trans_copy_iter(&copy, iter); 129 + bch2_trans_copy_iter(trans, &copy, iter); 130 130 131 - for_each_btree_key_max_continue_norestart(copy, insert->k.p, 0, k, ret) { 131 + for_each_btree_key_max_continue_norestart(trans, copy, insert->k.p, 0, k, ret) { 132 132 unsigned offset = 0; 133 133 134 134 if (bkey_gt(bkey_start_pos(&insert->k), bkey_start_pos(k.k)))

+3 -3

fs/bcachefs/fs-io-buffered.c

··· 183 183 if (ret) 184 184 goto err; 185 185 186 - bch2_btree_iter_set_snapshot(&iter, snapshot); 186 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 187 187 188 - bch2_btree_iter_set_pos(&iter, 188 + bch2_btree_iter_set_pos(trans, &iter, 189 189 POS(inum.inum, rbio->bio.bi_iter.bi_sector)); 190 190 191 - k = bch2_btree_iter_peek_slot(&iter); 191 + k = bch2_btree_iter_peek_slot(trans, &iter); 192 192 ret = bkey_err(k); 193 193 if (ret) 194 194 goto err;

+7 -7

fs/bcachefs/fs-io.c

··· 48 48 struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio); 49 49 50 50 closure_put(bio->cl); 51 - percpu_ref_put(&bio->ca->io_ref); 51 + percpu_ref_put(&bio->ca->io_ref[WRITE]); 52 52 bio_put(&bio->bio); 53 53 } 54 54 ··· 71 71 for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) { 72 72 rcu_read_lock(); 73 73 ca = rcu_dereference(c->devs[dev]); 74 - if (ca && !percpu_ref_tryget(&ca->io_ref)) 74 + if (ca && !percpu_ref_tryget(&ca->io_ref[WRITE])) 75 75 ca = NULL; 76 76 rcu_read_unlock(); 77 77 ··· 636 636 if (ret) 637 637 goto bkey_err; 638 638 639 - bch2_btree_iter_set_snapshot(&iter, snapshot); 639 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 640 640 641 - k = bch2_btree_iter_peek_slot(&iter); 641 + k = bch2_btree_iter_peek_slot(trans, &iter); 642 642 if ((ret = bkey_err(k))) 643 643 goto bkey_err; 644 644 ··· 649 649 /* already reserved */ 650 650 if (bkey_extent_is_reservation(k) && 651 651 bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) { 652 - bch2_btree_iter_advance(&iter); 652 + bch2_btree_iter_advance(trans, &iter); 653 653 continue; 654 654 } 655 655 656 656 if (bkey_extent_is_data(k.k) && 657 657 !(mode & FALLOC_FL_ZERO_RANGE)) { 658 - bch2_btree_iter_advance(&iter); 658 + bch2_btree_iter_advance(trans, &iter); 659 659 continue; 660 660 } 661 661 ··· 676 676 if (ret) 677 677 goto bkey_err; 678 678 } 679 - bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, hole_start)); 679 + bch2_btree_iter_set_pos(trans, &iter, POS(iter.pos.inode, hole_start)); 680 680 681 681 if (ret) 682 682 goto bkey_err;

+12 -12

fs/bcachefs/fs.c

··· 88 88 void *p, unsigned fields) 89 89 { 90 90 struct btree_trans *trans = bch2_trans_get(c); 91 - struct btree_iter iter = { NULL }; 91 + struct btree_iter iter = {}; 92 92 struct bch_inode_unpacked inode_u; 93 93 int ret; 94 94 retry: ··· 1075 1075 struct bch_fs *c = inode->v.i_sb->s_fs_info; 1076 1076 struct bch_qid qid; 1077 1077 struct btree_trans *trans; 1078 - struct btree_iter inode_iter = { NULL }; 1078 + struct btree_iter inode_iter = {}; 1079 1079 struct bch_inode_unpacked inode_u; 1080 1080 struct posix_acl *acl = NULL; 1081 1081 kuid_t kuid; ··· 1330 1330 if (ret) 1331 1331 continue; 1332 1332 1333 - bch2_btree_iter_set_snapshot(&iter, snapshot); 1333 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 1334 1334 1335 - k = bch2_btree_iter_peek_max(&iter, end); 1335 + k = bch2_btree_iter_peek_max(trans, &iter, end); 1336 1336 ret = bkey_err(k); 1337 1337 if (ret) 1338 1338 continue; ··· 1342 1342 1343 1343 if (!bkey_extent_is_data(k.k) && 1344 1344 k.k->type != KEY_TYPE_reservation) { 1345 - bch2_btree_iter_advance(&iter); 1345 + bch2_btree_iter_advance(trans, &iter); 1346 1346 continue; 1347 1347 } 1348 1348 ··· 1380 1380 bkey_copy(prev.k, cur.k); 1381 1381 have_extent = true; 1382 1382 1383 - bch2_btree_iter_set_pos(&iter, 1383 + bch2_btree_iter_set_pos(trans, &iter, 1384 1384 POS(iter.pos.inode, iter.pos.offset + sectors)); 1385 1385 } 1386 1386 bch2_trans_iter_exit(trans, &iter); ··· 1697 1697 if (ret) 1698 1698 goto err; 1699 1699 1700 - bch2_btree_iter_set_snapshot(&iter1, snapshot); 1701 - bch2_btree_iter_set_snapshot(&iter2, snapshot); 1700 + bch2_btree_iter_set_snapshot(trans, &iter1, snapshot); 1701 + bch2_btree_iter_set_snapshot(trans, &iter2, snapshot); 1702 1702 1703 1703 ret = bch2_inode_find_by_inum_trans(trans, inode_inum(inode), &inode_u); 1704 1704 if (ret) 1705 1705 goto err; 1706 1706 1707 1707 if (inode_u.bi_dir == dir->ei_inode.bi_inum) { 1708 - bch2_btree_iter_set_pos(&iter1, POS(inode_u.bi_dir, inode_u.bi_dir_offset)); 1708 + bch2_btree_iter_set_pos(trans, &iter1, POS(inode_u.bi_dir, inode_u.bi_dir_offset)); 1709 1709 1710 - k = bch2_btree_iter_peek_slot(&iter1); 1710 + k = bch2_btree_iter_peek_slot(trans, &iter1); 1711 1711 ret = bkey_err(k); 1712 1712 if (ret) 1713 1713 goto err; ··· 1731 1731 * File with multiple hardlinks and our backref is to the wrong 1732 1732 * directory - linear search: 1733 1733 */ 1734 - for_each_btree_key_continue_norestart(iter2, 0, k, ret) { 1734 + for_each_btree_key_continue_norestart(trans, iter2, 0, k, ret) { 1735 1735 if (k.k->p.inode > dir->ei_inode.bi_inum) 1736 1736 break; 1737 1737 ··· 2237 2237 /* XXX: create an anonymous device for multi device filesystems */ 2238 2238 sb->s_bdev = bdev; 2239 2239 sb->s_dev = bdev->bd_dev; 2240 - percpu_ref_put(&ca->io_ref); 2240 + percpu_ref_put(&ca->io_ref[READ]); 2241 2241 break; 2242 2242 } 2243 2243

+15 -15

fs/bcachefs/fsck.c

··· 186 186 { 187 187 struct bch_fs *c = trans->c; 188 188 struct qstr lostfound_str = QSTR("lost+found"); 189 - struct btree_iter lostfound_iter = { NULL }; 189 + struct btree_iter lostfound_iter = {}; 190 190 u64 inum = 0; 191 191 unsigned d_type = 0; 192 192 int ret; ··· 295 295 if (ret) 296 296 goto err; 297 297 298 - bch2_btree_iter_set_snapshot(&lostfound_iter, snapshot); 299 - ret = bch2_btree_iter_traverse(&lostfound_iter); 298 + bch2_btree_iter_set_snapshot(trans, &lostfound_iter, snapshot); 299 + ret = bch2_btree_iter_traverse(trans, &lostfound_iter); 300 300 if (ret) 301 301 goto err; 302 302 ··· 544 544 new_inode.bi_subvol = subvolid; 545 545 546 546 int ret = bch2_inode_create(trans, &inode_iter, &new_inode, snapshotid, cpu) ?: 547 - bch2_btree_iter_traverse(&inode_iter) ?: 547 + bch2_btree_iter_traverse(trans, &inode_iter) ?: 548 548 bch2_inode_write(trans, &inode_iter, &new_inode); 549 549 bch2_trans_iter_exit(trans, &inode_iter); 550 550 if (ret) ··· 609 609 struct btree_iter iter = {}; 610 610 611 611 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(inum, U64_MAX, snapshot), 0); 612 - struct bkey_s_c k = bch2_btree_iter_peek_prev_min(&iter, POS(inum, 0)); 612 + struct bkey_s_c k = bch2_btree_iter_peek_prev_min(trans, &iter, POS(inum, 0)); 613 613 bch2_trans_iter_exit(trans, &iter); 614 614 int ret = bkey_err(k); 615 615 if (ret) ··· 1557 1557 { 1558 1558 struct bch_fs *c = trans->c; 1559 1559 struct printbuf buf = PRINTBUF; 1560 - struct btree_iter iter1, iter2 = { NULL }; 1560 + struct btree_iter iter1, iter2 = {}; 1561 1561 struct bkey_s_c k1, k2; 1562 1562 int ret; 1563 1563 ··· 1566 1566 bch2_trans_iter_init(trans, &iter1, btree, pos1, 1567 1567 BTREE_ITER_all_snapshots| 1568 1568 BTREE_ITER_not_extents); 1569 - k1 = bch2_btree_iter_peek_max(&iter1, POS(pos1.inode, U64_MAX)); 1569 + k1 = bch2_btree_iter_peek_max(trans, &iter1, POS(pos1.inode, U64_MAX)); 1570 1570 ret = bkey_err(k1); 1571 1571 if (ret) 1572 1572 goto err; ··· 1586 1586 goto err; 1587 1587 } 1588 1588 1589 - bch2_trans_copy_iter(&iter2, &iter1); 1589 + bch2_trans_copy_iter(trans, &iter2, &iter1); 1590 1590 1591 1591 while (1) { 1592 - bch2_btree_iter_advance(&iter2); 1592 + bch2_btree_iter_advance(trans, &iter2); 1593 1593 1594 - k2 = bch2_btree_iter_peek_max(&iter2, POS(pos1.inode, U64_MAX)); 1594 + k2 = bch2_btree_iter_peek_max(trans, &iter2, POS(pos1.inode, U64_MAX)); 1595 1595 ret = bkey_err(k2); 1596 1596 if (ret) 1597 1597 goto err; ··· 1791 1791 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 1792 1792 struct btree_iter iter2; 1793 1793 1794 - bch2_trans_copy_iter(&iter2, iter); 1795 - bch2_btree_iter_set_snapshot(&iter2, i->snapshot); 1796 - ret = bch2_btree_iter_traverse(&iter2) ?: 1794 + bch2_trans_copy_iter(trans, &iter2, iter); 1795 + bch2_btree_iter_set_snapshot(trans, &iter2, i->snapshot); 1796 + ret = bch2_btree_iter_traverse(trans, &iter2) ?: 1797 1797 bch2_btree_delete_at(trans, &iter2, 1798 1798 BTREE_UPDATE_internal_snapshot_node); 1799 1799 bch2_trans_iter_exit(trans, &iter2); ··· 2185 2185 BTREE_ID_dirents, 2186 2186 SPOS(k.k->p.inode, k.k->p.offset, *i), 2187 2187 BTREE_ITER_intent); 2188 - ret = bch2_btree_iter_traverse(&delete_iter) ?: 2188 + ret = bch2_btree_iter_traverse(trans, &delete_iter) ?: 2189 2189 bch2_hash_delete_at(trans, bch2_dirent_hash_desc, 2190 2190 hash_info, 2191 2191 &delete_iter, ··· 2412 2412 bch2_trans_iter_exit(trans, &parent_iter); 2413 2413 bch2_trans_iter_init(trans, &parent_iter, 2414 2414 BTREE_ID_subvolumes, POS(0, parent), 0); 2415 - k = bch2_btree_iter_peek_slot(&parent_iter); 2415 + k = bch2_btree_iter_peek_slot(trans, &parent_iter); 2416 2416 ret = bkey_err(k); 2417 2417 if (ret) 2418 2418 goto err;

+9 -9

fs/bcachefs/inode.c

··· 940 940 BTREE_ITER_intent); 941 941 struct bkey_s_c k; 942 942 again: 943 - while ((k = bch2_btree_iter_peek(iter)).k && 943 + while ((k = bch2_btree_iter_peek(trans, iter)).k && 944 944 !(ret = bkey_err(k)) && 945 945 bkey_lt(k.k->p, POS(0, max))) { 946 946 if (pos < iter->pos.offset) ··· 951 951 * we've found just one: 952 952 */ 953 953 pos = iter->pos.offset + 1; 954 - bch2_btree_iter_set_pos(iter, POS(0, pos)); 954 + bch2_btree_iter_set_pos(trans, iter, POS(0, pos)); 955 955 } 956 956 957 957 if (!ret && pos < max) ··· 967 967 968 968 /* Retry from start */ 969 969 pos = start = min; 970 - bch2_btree_iter_set_pos(iter, POS(0, pos)); 970 + bch2_btree_iter_set_pos(trans, iter, POS(0, pos)); 971 971 le32_add_cpu(&cursor->v.gen, 1); 972 972 goto again; 973 973 found_slot: 974 - bch2_btree_iter_set_pos(iter, SPOS(0, pos, snapshot)); 975 - k = bch2_btree_iter_peek_slot(iter); 974 + bch2_btree_iter_set_pos(trans, iter, SPOS(0, pos, snapshot)); 975 + k = bch2_btree_iter_peek_slot(trans, iter); 976 976 ret = bkey_err(k); 977 977 if (ret) { 978 978 bch2_trans_iter_exit(trans, iter); ··· 1009 1009 if (ret) 1010 1010 goto err; 1011 1011 1012 - bch2_btree_iter_set_snapshot(&iter, snapshot); 1012 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 1013 1013 1014 - k = bch2_btree_iter_peek_max(&iter, end); 1014 + k = bch2_btree_iter_peek_max(trans, &iter, end); 1015 1015 ret = bkey_err(k); 1016 1016 if (ret) 1017 1017 goto err; ··· 1042 1042 int bch2_inode_rm(struct bch_fs *c, subvol_inum inum) 1043 1043 { 1044 1044 struct btree_trans *trans = bch2_trans_get(c); 1045 - struct btree_iter iter = { NULL }; 1045 + struct btree_iter iter = {}; 1046 1046 struct bkey_s_c k; 1047 1047 u32 snapshot; 1048 1048 int ret; ··· 1207 1207 static noinline int __bch2_inode_rm_snapshot(struct btree_trans *trans, u64 inum, u32 snapshot) 1208 1208 { 1209 1209 struct bch_fs *c = trans->c; 1210 - struct btree_iter iter = { NULL }; 1210 + struct btree_iter iter = {}; 1211 1211 struct bkey_i_inode_generation delete; 1212 1212 struct bch_inode_unpacked inode_u; 1213 1213 struct bkey_s_c k;

+9 -9

fs/bcachefs/io_misc.c

··· 43 43 bch2_bkey_buf_init(&new); 44 44 closure_init_stack(&cl); 45 45 46 - k = bch2_btree_iter_peek_slot(iter); 46 + k = bch2_btree_iter_peek_slot(trans, iter); 47 47 ret = bkey_err(k); 48 48 if (ret) 49 49 return ret; ··· 164 164 if (ret) 165 165 continue; 166 166 167 - bch2_btree_iter_set_snapshot(iter, snapshot); 167 + bch2_btree_iter_set_snapshot(trans, iter, snapshot); 168 168 169 169 /* 170 170 * peek_max() doesn't have ideal semantics for extents: 171 171 */ 172 - k = bch2_btree_iter_peek_max(iter, end_pos); 172 + k = bch2_btree_iter_peek_max(trans, iter, end_pos); 173 173 if (!k.k) 174 174 break; 175 175 ··· 230 230 u64 new_i_size, 231 231 bool warn) 232 232 { 233 - struct btree_iter iter = { NULL }; 233 + struct btree_iter iter = {}; 234 234 struct bch_inode_unpacked inode_u; 235 235 int ret; 236 236 ··· 399 399 if (ret) 400 400 goto err; 401 401 } else { 402 - bch2_btree_iter_set_pos(&iter, POS(inum.inum, src_offset)); 402 + bch2_btree_iter_set_pos(trans, &iter, POS(inum.inum, src_offset)); 403 403 404 404 ret = bch2_fpunch_at(trans, &iter, inum, src_offset + len, i_sectors_delta); 405 405 if (ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart)) ··· 425 425 if (ret) 426 426 goto btree_err; 427 427 428 - bch2_btree_iter_set_snapshot(&iter, snapshot); 429 - bch2_btree_iter_set_pos(&iter, SPOS(inum.inum, pos, snapshot)); 428 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 429 + bch2_btree_iter_set_pos(trans, &iter, SPOS(inum.inum, pos, snapshot)); 430 430 431 431 k = insert 432 - ? bch2_btree_iter_peek_prev_min(&iter, POS(inum.inum, 0)) 433 - : bch2_btree_iter_peek_max(&iter, POS(inum.inum, U64_MAX)); 432 + ? bch2_btree_iter_peek_prev_min(trans, &iter, POS(inum.inum, 0)) 433 + : bch2_btree_iter_peek_max(trans, &iter, POS(inum.inum, U64_MAX)); 434 434 if ((ret = bkey_err(k))) 435 435 goto btree_err; 436 436

+7 -7

fs/bcachefs/io_read.c

··· 394 394 395 395 if (rbio->have_ioref) { 396 396 struct bch_dev *ca = bch2_dev_have_ref(rbio->c, rbio->pick.ptr.dev); 397 - percpu_ref_put(&ca->io_ref); 397 + percpu_ref_put(&ca->io_ref[READ]); 398 398 } 399 399 400 400 if (rbio->split) { ··· 909 909 910 910 prt_printf(&buf, "memory gen: %u", gen); 911 911 912 - ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(&iter))); 912 + ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(trans, &iter))); 913 913 if (!ret) { 914 914 prt_newline(&buf); 915 915 bch2_bkey_val_to_text(&buf, c, k); ··· 1003 1003 unlikely(dev_ptr_stale(ca, &pick.ptr))) { 1004 1004 read_from_stale_dirty_pointer(trans, ca, k, pick.ptr); 1005 1005 bch2_mark_io_failure(failed, &pick, false); 1006 - percpu_ref_put(&ca->io_ref); 1006 + percpu_ref_put(&ca->io_ref[READ]); 1007 1007 goto retry_pick; 1008 1008 } 1009 1009 ··· 1036 1036 */ 1037 1037 if (pick.crc.compressed_size > u->op.wbio.bio.bi_iter.bi_size) { 1038 1038 if (ca) 1039 - percpu_ref_put(&ca->io_ref); 1039 + percpu_ref_put(&ca->io_ref[READ]); 1040 1040 rbio->ret = -BCH_ERR_data_read_buffer_too_small; 1041 1041 goto out_read_done; 1042 1042 } ··· 1285 1285 if (ret) 1286 1286 goto err; 1287 1287 1288 - bch2_btree_iter_set_snapshot(&iter, snapshot); 1288 + bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 1289 1289 1290 - bch2_btree_iter_set_pos(&iter, 1290 + bch2_btree_iter_set_pos(trans, &iter, 1291 1291 POS(inum.inum, bvec_iter.bi_sector)); 1292 1292 1293 - k = bch2_btree_iter_peek_slot(&iter); 1293 + k = bch2_btree_iter_peek_slot(trans, &iter); 1294 1294 ret = bkey_err(k); 1295 1295 if (ret) 1296 1296 goto err;

+26 -14

fs/bcachefs/io_write.c

··· 168 168 *i_sectors_delta = 0; 169 169 *disk_sectors_delta = 0; 170 170 171 - bch2_trans_copy_iter(&iter, extent_iter); 171 + bch2_trans_copy_iter(trans, &iter, extent_iter); 172 172 173 - for_each_btree_key_max_continue_norestart(iter, 173 + for_each_btree_key_max_continue_norestart(trans, iter, 174 174 new->k.p, BTREE_ITER_slots, old, ret) { 175 175 s64 sectors = min(new->k.p.offset, old.k->p.offset) - 176 176 max(bkey_start_offset(&new->k), ··· 292 292 * path already traversed at iter->pos because 293 293 * bch2_trans_extent_update() will use it to attempt extent merging 294 294 */ 295 - ret = __bch2_btree_iter_traverse(iter); 295 + ret = __bch2_btree_iter_traverse(trans, iter); 296 296 if (ret) 297 297 return ret; 298 298 ··· 337 337 338 338 if (i_sectors_delta_total) 339 339 *i_sectors_delta_total += i_sectors_delta; 340 - bch2_btree_iter_set_pos(iter, next_pos); 340 + bch2_btree_iter_set_pos(trans, iter, next_pos); 341 341 return 0; 342 342 } 343 343 ··· 445 445 BUG_ON(c->opts.nochanges); 446 446 447 447 bkey_for_each_ptr(ptrs, ptr) { 448 + /* 449 + * XXX: btree writes should be using io_ref[WRITE], but we 450 + * aren't retrying failed btree writes yet (due to device 451 + * removal/ro): 452 + */ 448 453 struct bch_dev *ca = nocow 449 454 ? bch2_dev_have_ref(c, ptr->dev) 450 455 : bch2_dev_get_ioref(c, ptr->dev, type == BCH_DATA_btree ? READ : WRITE); ··· 702 697 bch2_account_io_completion(ca, BCH_MEMBER_ERROR_write, 703 698 wbio->submit_time, !bio->bi_status); 704 699 705 - if (bio->bi_status) { 706 - bch_err_inum_offset_ratelimited(ca, 707 - op->pos.inode, 708 - wbio->inode_offset << 9, 709 - "data write error: %s", 710 - bch2_blk_status_to_str(bio->bi_status)); 700 + if (unlikely(bio->bi_status)) { 701 + if (ca) 702 + bch_err_inum_offset_ratelimited(ca, 703 + op->pos.inode, 704 + wbio->inode_offset << 9, 705 + "data write error: %s", 706 + bch2_blk_status_to_str(bio->bi_status)); 707 + else 708 + bch_err_inum_offset_ratelimited(c, 709 + op->pos.inode, 710 + wbio->inode_offset << 9, 711 + "data write error: %s", 712 + bch2_blk_status_to_str(bio->bi_status)); 711 713 set_bit(wbio->dev, op->failed.d); 712 714 op->flags |= BCH_WRITE_io_error; 713 715 } ··· 727 715 } 728 716 729 717 if (wbio->have_ioref) 730 - percpu_ref_put(&ca->io_ref); 718 + percpu_ref_put(&ca->io_ref[WRITE]); 731 719 732 720 if (wbio->bounce) 733 721 bch2_bio_free_pages_pool(c, bio); ··· 1305 1293 if (ret) 1306 1294 break; 1307 1295 1308 - k = bch2_btree_iter_peek_slot(&iter); 1296 + k = bch2_btree_iter_peek_slot(trans, &iter); 1309 1297 ret = bkey_err(k); 1310 1298 if (ret) 1311 1299 break; ··· 1389 1377 bch2_keylist_push(&op->insert_keys); 1390 1378 if (op->flags & BCH_WRITE_submitted) 1391 1379 break; 1392 - bch2_btree_iter_advance(&iter); 1380 + bch2_btree_iter_advance(trans, &iter); 1393 1381 } 1394 1382 out: 1395 1383 bch2_trans_iter_exit(trans, &iter); ··· 1426 1414 return; 1427 1415 err_get_ioref: 1428 1416 darray_for_each(buckets, i) 1429 - percpu_ref_put(&bch2_dev_have_ref(c, i->b.inode)->io_ref); 1417 + percpu_ref_put(&bch2_dev_have_ref(c, i->b.inode)->io_ref[WRITE]); 1430 1418 1431 1419 /* Fall back to COW path: */ 1432 1420 goto out;

+10 -4

fs/bcachefs/journal.c

··· 1315 1315 1316 1316 int ret = bch2_dev_journal_alloc(ca, true); 1317 1317 if (ret) { 1318 - percpu_ref_put(&ca->io_ref); 1318 + percpu_ref_put(&ca->io_ref[READ]); 1319 1319 return ret; 1320 1320 } 1321 1321 } ··· 1404 1404 1405 1405 nr = cur_seq - last_seq; 1406 1406 1407 + /* 1408 + * Extra fudge factor, in case we crashed when the journal pin fifo was 1409 + * nearly or completely full. We'll need to be able to open additional 1410 + * journal entries (at least a few) in order for journal replay to get 1411 + * going: 1412 + */ 1413 + nr += nr / 4; 1414 + 1407 1415 if (nr + 1 > j->pin.size) { 1408 1416 free_fifo(&j->pin); 1409 1417 init_fifo(&j->pin, roundup_pow_of_two(nr + 1), GFP_KERNEL); ··· 1469 1461 j->reservations.idx = journal_cur_seq(j); 1470 1462 1471 1463 c->last_bucket_seq_cleanup = journal_cur_seq(j); 1472 - 1473 - bch2_journal_space_available(j); 1474 1464 spin_unlock(&j->lock); 1475 1465 1476 - return bch2_journal_reclaim_start(j); 1466 + return 0; 1477 1467 } 1478 1468 1479 1469 /* init/exit: */

+4 -4

fs/bcachefs/journal_io.c

··· 1218 1218 out: 1219 1219 bch_verbose(c, "journal read done on device %s, ret %i", ca->name, ret); 1220 1220 kvfree(buf.data); 1221 - percpu_ref_put(&ca->io_ref); 1221 + percpu_ref_put(&ca->io_ref[READ]); 1222 1222 closure_return(cl); 1223 1223 return; 1224 1224 err: ··· 1253 1253 1254 1254 if ((ca->mi.state == BCH_MEMBER_STATE_rw || 1255 1255 ca->mi.state == BCH_MEMBER_STATE_ro) && 1256 - percpu_ref_tryget(&ca->io_ref)) 1256 + percpu_ref_tryget(&ca->io_ref[READ])) 1257 1257 closure_call(&ca->journal.read, 1258 1258 bch2_journal_read_device, 1259 1259 system_unbound_wq, ··· 1768 1768 } 1769 1769 1770 1770 closure_put(&w->io); 1771 - percpu_ref_put(&ca->io_ref); 1771 + percpu_ref_put(&ca->io_ref[WRITE]); 1772 1772 } 1773 1773 1774 1774 static CLOSURE_CALLBACK(journal_write_submit) ··· 1843 1843 1844 1844 if (w->separate_flush) { 1845 1845 for_each_rw_member(c, ca) { 1846 - percpu_ref_get(&ca->io_ref); 1846 + percpu_ref_get(&ca->io_ref[WRITE]); 1847 1847 1848 1848 struct journal_device *ja = &ca->journal; 1849 1849 struct bio *bio = &ja->bio[w->idx]->bio;

+2 -2

fs/bcachefs/migrate.c

··· 130 130 retry: 131 131 ret = 0; 132 132 while (bch2_trans_begin(trans), 133 - (b = bch2_btree_iter_peek_node(&iter)) && 133 + (b = bch2_btree_iter_peek_node(trans, &iter)) && 134 134 !(ret = PTR_ERR_OR_ZERO(b))) { 135 135 bch2_progress_update_iter(trans, progress, &iter, "dropping metadata"); 136 136 ··· 154 154 if (ret) 155 155 break; 156 156 next: 157 - bch2_btree_iter_next_node(&iter); 157 + bch2_btree_iter_next_node(trans, &iter); 158 158 } 159 159 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 160 160 goto retry;

+7 -7

fs/bcachefs/move.c

··· 545 545 BTREE_ID_reflink, reflink_pos, 546 546 BTREE_ITER_not_extents); 547 547 548 - struct bkey_s_c k = bch2_btree_iter_peek(iter); 548 + struct bkey_s_c k = bch2_btree_iter_peek(trans, iter); 549 549 if (!k.k || bkey_err(k)) { 550 550 bch2_trans_iter_exit(trans, iter); 551 551 return k; ··· 603 603 604 604 bch2_trans_begin(trans); 605 605 606 - k = bch2_btree_iter_peek(&iter); 606 + k = bch2_btree_iter_peek(trans, &iter); 607 607 if (!k.k) 608 608 break; 609 609 ··· 681 681 if (ctxt->stats) 682 682 atomic64_add(k.k->size, &ctxt->stats->sectors_seen); 683 683 next_nondata: 684 - bch2_btree_iter_advance(&iter); 684 + bch2_btree_iter_advance(trans, &iter); 685 685 } 686 686 687 687 bch2_trans_iter_exit(trans, &reflink_iter); ··· 794 794 795 795 bch2_trans_begin(trans); 796 796 797 - k = bch2_btree_iter_peek(&bp_iter); 797 + k = bch2_btree_iter_peek(trans, &bp_iter); 798 798 ret = bkey_err(k); 799 799 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 800 800 continue; ··· 876 876 if (ctxt->stats) 877 877 atomic64_add(sectors, &ctxt->stats->sectors_seen); 878 878 next: 879 - bch2_btree_iter_advance(&bp_iter); 879 + bch2_btree_iter_advance(trans, &bp_iter); 880 880 } 881 881 err: 882 882 bch2_trans_iter_exit(trans, &bp_iter); ··· 991 991 retry: 992 992 ret = 0; 993 993 while (bch2_trans_begin(trans), 994 - (b = bch2_btree_iter_peek_node(&iter)) && 994 + (b = bch2_btree_iter_peek_node(trans, &iter)) && 995 995 !(ret = PTR_ERR_OR_ZERO(b))) { 996 996 if (kthread && kthread_should_stop()) 997 997 break; ··· 1011 1011 if (ret) 1012 1012 break; 1013 1013 next: 1014 - bch2_btree_iter_next_node(&iter); 1014 + bch2_btree_iter_next_node(trans, &iter); 1015 1015 } 1016 1016 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 1017 1017 goto retry;

+6 -2

fs/bcachefs/movinggc.c

··· 280 280 s64 wait = S64_MAX, fragmented_allowed, fragmented; 281 281 282 282 for_each_rw_member(c, ca) { 283 - struct bch_dev_usage usage = bch2_dev_usage_read(ca); 283 + struct bch_dev_usage_full usage_full = bch2_dev_usage_full_read(ca); 284 + struct bch_dev_usage usage; 285 + 286 + for (unsigned i = 0; i < BCH_DATA_NR; i++) 287 + usage.buckets[i] = usage_full.d[i].buckets; 284 288 285 289 fragmented_allowed = ((__dev_buckets_available(ca, usage, BCH_WATERMARK_stripe) * 286 290 ca->mi.bucket_size) >> 1); ··· 292 288 293 289 for (unsigned i = 0; i < BCH_DATA_NR; i++) 294 290 if (data_type_movable(i)) 295 - fragmented += usage.d[i].fragmented; 291 + fragmented += usage_full.d[i].fragmented; 296 292 297 293 wait = min(wait, max(0LL, fragmented_allowed - fragmented)); 298 294 }

+19 -19

fs/bcachefs/namei.c

··· 28 28 unsigned flags) 29 29 { 30 30 struct bch_fs *c = trans->c; 31 - struct btree_iter dir_iter = { NULL }; 32 - struct btree_iter inode_iter = { NULL }; 31 + struct btree_iter dir_iter = {}; 32 + struct btree_iter inode_iter = {}; 33 33 subvol_inum new_inum = dir; 34 34 u64 now = bch2_current_time(c); 35 35 u64 cpu = raw_smp_processor_id(); ··· 127 127 if (ret) 128 128 goto err; 129 129 130 - bch2_btree_iter_set_snapshot(&dir_iter, dir_snapshot); 131 - ret = bch2_btree_iter_traverse(&dir_iter); 130 + bch2_btree_iter_set_snapshot(trans, &dir_iter, dir_snapshot); 131 + ret = bch2_btree_iter_traverse(trans, &dir_iter); 132 132 if (ret) 133 133 goto err; 134 134 } ··· 177 177 new_inode->bi_depth = dir_u->bi_depth + 1; 178 178 179 179 inode_iter.flags &= ~BTREE_ITER_all_snapshots; 180 - bch2_btree_iter_set_snapshot(&inode_iter, snapshot); 180 + bch2_btree_iter_set_snapshot(trans, &inode_iter, snapshot); 181 181 182 - ret = bch2_btree_iter_traverse(&inode_iter) ?: 182 + ret = bch2_btree_iter_traverse(trans, &inode_iter) ?: 183 183 bch2_inode_write(trans, &inode_iter, new_inode); 184 184 err: 185 185 bch2_trans_iter_exit(trans, &inode_iter); ··· 193 193 const struct qstr *name) 194 194 { 195 195 struct bch_fs *c = trans->c; 196 - struct btree_iter dir_iter = { NULL }; 197 - struct btree_iter inode_iter = { NULL }; 196 + struct btree_iter dir_iter = {}; 197 + struct btree_iter inode_iter = {}; 198 198 struct bch_hash_info dir_hash; 199 199 u64 now = bch2_current_time(c); 200 200 u64 dir_offset = 0; ··· 253 253 bool deleting_subvol) 254 254 { 255 255 struct bch_fs *c = trans->c; 256 - struct btree_iter dir_iter = { NULL }; 257 - struct btree_iter dirent_iter = { NULL }; 258 - struct btree_iter inode_iter = { NULL }; 256 + struct btree_iter dir_iter = {}; 257 + struct btree_iter dirent_iter = {}; 258 + struct btree_iter inode_iter = {}; 259 259 struct bch_hash_info dir_hash; 260 260 subvol_inum inum; 261 261 u64 now = bch2_current_time(c); ··· 301 301 if (ret) 302 302 goto err; 303 303 304 - k = bch2_btree_iter_peek_slot(&dirent_iter); 304 + k = bch2_btree_iter_peek_slot(trans, &dirent_iter); 305 305 ret = bkey_err(k); 306 306 if (ret) 307 307 goto err; ··· 310 310 * If we're deleting a subvolume, we need to really delete the 311 311 * dirent, not just emit a whiteout in the current snapshot: 312 312 */ 313 - bch2_btree_iter_set_snapshot(&dirent_iter, k.k->p.snapshot); 314 - ret = bch2_btree_iter_traverse(&dirent_iter); 313 + bch2_btree_iter_set_snapshot(trans, &dirent_iter, k.k->p.snapshot); 314 + ret = bch2_btree_iter_traverse(trans, &dirent_iter); 315 315 if (ret) 316 316 goto err; 317 317 } else { ··· 390 390 enum bch_rename_mode mode) 391 391 { 392 392 struct bch_fs *c = trans->c; 393 - struct btree_iter src_dir_iter = { NULL }; 394 - struct btree_iter dst_dir_iter = { NULL }; 395 - struct btree_iter src_inode_iter = { NULL }; 396 - struct btree_iter dst_inode_iter = { NULL }; 393 + struct btree_iter src_dir_iter = {}; 394 + struct btree_iter dst_dir_iter = {}; 395 + struct btree_iter src_inode_iter = {}; 396 + struct btree_iter dst_inode_iter = {}; 397 397 struct bch_hash_info src_hash, dst_hash; 398 398 subvol_inum src_inum, dst_inum; 399 399 u64 src_offset, dst_offset; ··· 666 666 { 667 667 struct bch_fs *c = trans->c; 668 668 struct printbuf buf = PRINTBUF; 669 - struct btree_iter bp_iter = { NULL }; 669 + struct btree_iter bp_iter = {}; 670 670 int ret = 0; 671 671 672 672 if (inode_points_to_dirent(target, d))

+1 -1

fs/bcachefs/quota.c

··· 516 516 bch2_quota_acct(c, bch_qid(&u), Q_INO, 1, 517 517 KEY_TYPE_QUOTA_NOCHECK); 518 518 advance: 519 - bch2_btree_iter_set_pos(iter, bpos_nosnap_successor(iter->pos)); 519 + bch2_btree_iter_set_pos(trans, iter, bpos_nosnap_successor(iter->pos)); 520 520 return 0; 521 521 } 522 522

+6 -6

fs/bcachefs/rebalance.c

··· 233 233 bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work, 234 234 SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX), 235 235 BTREE_ITER_intent); 236 - k = bch2_btree_iter_peek_slot(&iter); 236 + k = bch2_btree_iter_peek_slot(trans, &iter); 237 237 ret = bkey_err(k); 238 238 if (ret) 239 239 goto err; ··· 281 281 bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work, 282 282 SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX), 283 283 BTREE_ITER_intent); 284 - k = bch2_btree_iter_peek_slot(&iter); 284 + k = bch2_btree_iter_peek_slot(trans, &iter); 285 285 ret = bkey_err(k); 286 286 if (ret) 287 287 goto err; ··· 301 301 struct btree_iter *work_iter) 302 302 { 303 303 return !kthread_should_stop() 304 - ? bch2_btree_iter_peek(work_iter) 304 + ? bch2_btree_iter_peek(trans, work_iter) 305 305 : bkey_s_c_null; 306 306 } 307 307 ··· 335 335 work_pos.inode ? BTREE_ID_extents : BTREE_ID_reflink, 336 336 work_pos, 337 337 BTREE_ITER_all_snapshots); 338 - struct bkey_s_c k = bch2_btree_iter_peek_slot(extent_iter); 338 + struct bkey_s_c k = bch2_btree_iter_peek_slot(trans, extent_iter); 339 339 if (bkey_err(k)) 340 340 return k; 341 341 ··· 511 511 struct btree_trans *trans = ctxt->trans; 512 512 struct bch_fs *c = trans->c; 513 513 struct bch_fs_rebalance *r = &c->rebalance; 514 - struct btree_iter rebalance_work_iter, extent_iter = { NULL }; 514 + struct btree_iter rebalance_work_iter, extent_iter = {}; 515 515 struct bkey_s_c k; 516 516 int ret = 0; 517 517 ··· 552 552 if (ret) 553 553 break; 554 554 555 - bch2_btree_iter_advance(&rebalance_work_iter); 555 + bch2_btree_iter_advance(trans, &rebalance_work_iter); 556 556 } 557 557 558 558 bch2_trans_iter_exit(trans, &extent_iter);

+3 -3

fs/bcachefs/recovery.c

··· 198 198 bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p, 199 199 BTREE_MAX_DEPTH, k->level, 200 200 BTREE_ITER_intent); 201 - int ret = bch2_btree_iter_traverse(&iter); 201 + int ret = bch2_btree_iter_traverse(trans, &iter); 202 202 if (ret) 203 203 goto out; 204 204 ··· 261 261 bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p, 262 262 BTREE_MAX_DEPTH, k->level, 263 263 iter_flags); 264 - ret = bch2_btree_iter_traverse(&iter); 264 + ret = bch2_btree_iter_traverse(trans, &iter); 265 265 if (ret) 266 266 goto out; 267 267 ··· 270 270 bch2_trans_iter_exit(trans, &iter); 271 271 bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p, 272 272 BTREE_MAX_DEPTH, 0, iter_flags); 273 - ret = bch2_btree_iter_traverse(&iter) ?: 273 + ret = bch2_btree_iter_traverse(trans, &iter) ?: 274 274 bch2_btree_increase_depth(trans, iter.path, 0) ?: 275 275 -BCH_ERR_transaction_restart_nested; 276 276 goto out;

+12 -11

fs/bcachefs/reflink.c

··· 495 495 bool reflink_p_may_update_opts_field) 496 496 { 497 497 struct bch_fs *c = trans->c; 498 - struct btree_iter reflink_iter = { NULL }; 498 + struct btree_iter reflink_iter = {}; 499 499 struct bkey_s_c k; 500 500 struct bkey_i *r_v; 501 501 struct bkey_i_reflink_p *r_p; ··· 507 507 508 508 bch2_trans_iter_init(trans, &reflink_iter, BTREE_ID_reflink, POS_MAX, 509 509 BTREE_ITER_intent); 510 - k = bch2_btree_iter_peek_prev(&reflink_iter); 510 + k = bch2_btree_iter_peek_prev(trans, &reflink_iter); 511 511 ret = bkey_err(k); 512 512 if (ret) 513 513 goto err; ··· 569 569 return ret; 570 570 } 571 571 572 - static struct bkey_s_c get_next_src(struct btree_iter *iter, struct bpos end) 572 + static struct bkey_s_c get_next_src(struct btree_trans *trans, 573 + struct btree_iter *iter, struct bpos end) 573 574 { 574 575 struct bkey_s_c k; 575 576 int ret; 576 577 577 - for_each_btree_key_max_continue_norestart(*iter, end, 0, k, ret) { 578 + for_each_btree_key_max_continue_norestart(trans, *iter, end, 0, k, ret) { 578 579 if (bkey_extent_is_unwritten(k)) 579 580 continue; 580 581 ··· 584 583 } 585 584 586 585 if (bkey_ge(iter->pos, end)) 587 - bch2_btree_iter_set_pos(iter, end); 586 + bch2_btree_iter_set_pos(trans, iter, end); 588 587 return ret ? bkey_s_c_err(ret) : bkey_s_c_null; 589 588 } 590 589 ··· 648 647 if (ret) 649 648 continue; 650 649 651 - bch2_btree_iter_set_snapshot(&src_iter, src_snapshot); 650 + bch2_btree_iter_set_snapshot(trans, &src_iter, src_snapshot); 652 651 653 652 ret = bch2_subvolume_get_snapshot(trans, dst_inum.subvol, 654 653 &dst_snapshot); 655 654 if (ret) 656 655 continue; 657 656 658 - bch2_btree_iter_set_snapshot(&dst_iter, dst_snapshot); 657 + bch2_btree_iter_set_snapshot(trans, &dst_iter, dst_snapshot); 659 658 660 659 if (dst_inum.inum < src_inum.inum) { 661 660 /* Avoid some lock cycle transaction restarts */ 662 - ret = bch2_btree_iter_traverse(&dst_iter); 661 + ret = bch2_btree_iter_traverse(trans, &dst_iter); 663 662 if (ret) 664 663 continue; 665 664 } 666 665 667 666 dst_done = dst_iter.pos.offset - dst_start.offset; 668 667 src_want = POS(src_start.inode, src_start.offset + dst_done); 669 - bch2_btree_iter_set_pos(&src_iter, src_want); 668 + bch2_btree_iter_set_pos(trans, &src_iter, src_want); 670 669 671 - src_k = get_next_src(&src_iter, src_end); 670 + src_k = get_next_src(trans, &src_iter, src_end); 672 671 ret = bkey_err(src_k); 673 672 if (ret) 674 673 continue; ··· 739 738 740 739 do { 741 740 struct bch_inode_unpacked inode_u; 742 - struct btree_iter inode_iter = { NULL }; 741 + struct btree_iter inode_iter = {}; 743 742 744 743 bch2_trans_begin(trans); 745 744

+12 -11

fs/bcachefs/sb-members.h

··· 20 20 21 21 static inline bool bch2_dev_is_online(struct bch_dev *ca) 22 22 { 23 - return !percpu_ref_is_zero(&ca->io_ref); 23 + return !percpu_ref_is_zero(&ca->io_ref[READ]); 24 24 } 25 25 26 26 static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *, unsigned); ··· 156 156 157 157 static inline struct bch_dev *bch2_get_next_online_dev(struct bch_fs *c, 158 158 struct bch_dev *ca, 159 - unsigned state_mask) 159 + unsigned state_mask, 160 + int rw) 160 161 { 161 162 rcu_read_lock(); 162 163 if (ca) 163 - percpu_ref_put(&ca->io_ref); 164 + percpu_ref_put(&ca->io_ref[rw]); 164 165 165 166 while ((ca = __bch2_next_dev(c, ca, NULL)) && 166 167 (!((1 << ca->mi.state) & state_mask) || 167 - !percpu_ref_tryget(&ca->io_ref))) 168 + !percpu_ref_tryget(&ca->io_ref[rw]))) 168 169 ; 169 170 rcu_read_unlock(); 170 171 171 172 return ca; 172 173 } 173 174 174 - #define __for_each_online_member(_c, _ca, state_mask) \ 175 + #define __for_each_online_member(_c, _ca, state_mask, rw) \ 175 176 for (struct bch_dev *_ca = NULL; \ 176 - (_ca = bch2_get_next_online_dev(_c, _ca, state_mask));) 177 + (_ca = bch2_get_next_online_dev(_c, _ca, state_mask, rw));) 177 178 178 179 #define for_each_online_member(c, ca) \ 179 - __for_each_online_member(c, ca, ~0) 180 + __for_each_online_member(c, ca, ~0, READ) 180 181 181 182 #define for_each_rw_member(c, ca) \ 182 - __for_each_online_member(c, ca, BIT(BCH_MEMBER_STATE_rw)) 183 + __for_each_online_member(c, ca, BIT(BCH_MEMBER_STATE_rw), WRITE) 183 184 184 185 #define for_each_readable_member(c, ca) \ 185 - __for_each_online_member(c, ca, BIT( BCH_MEMBER_STATE_rw)|BIT(BCH_MEMBER_STATE_ro)) 186 + __for_each_online_member(c, ca, BIT( BCH_MEMBER_STATE_rw)|BIT(BCH_MEMBER_STATE_ro), READ) 186 187 187 188 static inline bool bch2_dev_exists(const struct bch_fs *c, unsigned dev) 188 189 { ··· 288 287 289 288 rcu_read_lock(); 290 289 struct bch_dev *ca = bch2_dev_rcu(c, dev); 291 - if (ca && !percpu_ref_tryget(&ca->io_ref)) 290 + if (ca && !percpu_ref_tryget(&ca->io_ref[rw])) 292 291 ca = NULL; 293 292 rcu_read_unlock(); 294 293 ··· 298 297 return ca; 299 298 300 299 if (ca) 301 - percpu_ref_put(&ca->io_ref); 300 + percpu_ref_put(&ca->io_ref[rw]); 302 301 return NULL; 303 302 } 304 303

+5 -8

fs/bcachefs/snapshot.c

··· 843 843 { 844 844 struct bch_fs *c = trans->c; 845 845 846 - if (bch2_snapshot_exists(c, id)) 847 - return 0; 848 - 849 846 /* Do we need to reconstruct the snapshot_tree entry as well? */ 850 847 struct btree_iter iter; 851 848 struct bkey_s_c k; ··· 1071 1074 static int bch2_snapshot_node_delete(struct btree_trans *trans, u32 id) 1072 1075 { 1073 1076 struct bch_fs *c = trans->c; 1074 - struct btree_iter iter, p_iter = (struct btree_iter) { NULL }; 1075 - struct btree_iter c_iter = (struct btree_iter) { NULL }; 1076 - struct btree_iter tree_iter = (struct btree_iter) { NULL }; 1077 + struct btree_iter iter, p_iter = {}; 1078 + struct btree_iter c_iter = {}; 1079 + struct btree_iter tree_iter = {}; 1077 1080 struct bkey_s_c_snapshot s; 1078 1081 u32 parent_id, child_id; 1079 1082 unsigned i; ··· 1190 1193 1191 1194 bch2_trans_iter_init(trans, &iter, BTREE_ID_snapshots, 1192 1195 POS_MIN, BTREE_ITER_intent); 1193 - k = bch2_btree_iter_peek(&iter); 1196 + k = bch2_btree_iter_peek(trans, &iter); 1194 1197 ret = bkey_err(k); 1195 1198 if (ret) 1196 1199 goto err; 1197 1200 1198 1201 for (i = 0; i < nr_snapids; i++) { 1199 - k = bch2_btree_iter_prev_slot(&iter); 1202 + k = bch2_btree_iter_prev_slot(trans, &iter); 1200 1203 ret = bkey_err(k); 1201 1204 if (ret) 1202 1205 goto err;

+1 -1

fs/bcachefs/str_hash.c

··· 195 195 struct btree_iter *k_iter, struct bkey_s_c hash_k) 196 196 { 197 197 struct bch_fs *c = trans->c; 198 - struct btree_iter iter = { NULL }; 198 + struct btree_iter iter = {}; 199 199 struct printbuf buf = PRINTBUF; 200 200 struct bkey_s_c k; 201 201 int ret = 0;

+4 -4

fs/bcachefs/str_hash.h

··· 231 231 struct bkey_s_c k; 232 232 int ret; 233 233 234 - bch2_trans_copy_iter(&iter, start); 234 + bch2_trans_copy_iter(trans, &iter, start); 235 235 236 - bch2_btree_iter_advance(&iter); 236 + bch2_btree_iter_advance(trans, &iter); 237 237 238 - for_each_btree_key_continue_norestart(iter, BTREE_ITER_slots, k, ret) { 238 + for_each_btree_key_continue_norestart(trans, iter, BTREE_ITER_slots, k, ret) { 239 239 if (k.k->type != desc.key_type && 240 240 k.k->type != KEY_TYPE_hash_whiteout) 241 241 break; ··· 280 280 } 281 281 282 282 if (!slot.path && !(flags & STR_HASH_must_replace)) 283 - bch2_trans_copy_iter(&slot, iter); 283 + bch2_trans_copy_iter(trans, &slot, iter); 284 284 285 285 if (k.k->type != KEY_TYPE_hash_whiteout) 286 286 goto not_found;

+2 -2

fs/bcachefs/subvolume.c

··· 275 275 struct btree_iter iter; 276 276 277 277 bch2_trans_iter_init(trans, &iter, BTREE_ID_subvolume_children, POS(subvol, 0), 0); 278 - struct bkey_s_c k = bch2_btree_iter_peek(&iter); 278 + struct bkey_s_c k = bch2_btree_iter_peek(trans, &iter); 279 279 bch2_trans_iter_exit(trans, &iter); 280 280 281 281 return bkey_err(k) ?: k.k && k.k->p.inode == subvol ··· 574 574 bool ro) 575 575 { 576 576 struct bch_fs *c = trans->c; 577 - struct btree_iter dst_iter, src_iter = (struct btree_iter) { NULL }; 577 + struct btree_iter dst_iter, src_iter = {}; 578 578 struct bkey_i_subvolume *new_subvol = NULL; 579 579 struct bkey_i_subvolume *src_subvol = NULL; 580 580 u32 parent = 0, new_nodes[2], snapshot_subvols[2];

+7 -7

fs/bcachefs/subvolume.h

··· 33 33 int bch2_subvol_is_ro(struct bch_fs *, u32); 34 34 35 35 static inline struct bkey_s_c 36 - bch2_btree_iter_peek_in_subvolume_max_type(struct btree_iter *iter, struct bpos end, 37 - u32 subvolid, unsigned flags) 36 + bch2_btree_iter_peek_in_subvolume_max_type(struct btree_trans *trans, struct btree_iter *iter, 37 + struct bpos end, u32 subvolid, unsigned flags) 38 38 { 39 39 u32 snapshot; 40 - int ret = bch2_subvolume_get_snapshot(iter->trans, subvolid, &snapshot); 40 + int ret = bch2_subvolume_get_snapshot(trans, subvolid, &snapshot); 41 41 if (ret) 42 42 return bkey_s_c_err(ret); 43 43 44 - bch2_btree_iter_set_snapshot(iter, snapshot); 45 - return bch2_btree_iter_peek_max_type(iter, end, flags); 44 + bch2_btree_iter_set_snapshot(trans, iter, snapshot); 45 + return bch2_btree_iter_peek_max_type(trans, iter, end, flags); 46 46 } 47 47 48 48 #define for_each_btree_key_in_subvolume_max_continue(_trans, _iter, \ ··· 53 53 \ 54 54 do { \ 55 55 _ret3 = lockrestart_do(_trans, ({ \ 56 - (_k) = bch2_btree_iter_peek_in_subvolume_max_type(&(_iter), \ 56 + (_k) = bch2_btree_iter_peek_in_subvolume_max_type(trans, &(_iter),\ 57 57 _end, _subvolid, (_flags)); \ 58 58 if (!(_k).k) \ 59 59 break; \ 60 60 \ 61 61 bkey_err(_k) ?: (_do); \ 62 62 })); \ 63 - } while (!_ret3 && bch2_btree_iter_advance(&(_iter))); \ 63 + } while (!_ret3 && bch2_btree_iter_advance(_trans, &(_iter))); \ 64 64 \ 65 65 bch2_trans_iter_exit((_trans), &(_iter)); \ 66 66 _ret3; \

+14 -7

fs/bcachefs/super-io.c

··· 248 248 struct bch_sb_handle *dev_sb = &ca->disk_sb; 249 249 250 250 if (bch2_sb_realloc(dev_sb, le32_to_cpu(dev_sb->sb->u64s) + d)) { 251 - percpu_ref_put(&ca->io_ref); 251 + percpu_ref_put(&ca->io_ref[READ]); 252 252 return NULL; 253 253 } 254 254 } ··· 945 945 } 946 946 947 947 closure_put(&ca->fs->sb_write); 948 - percpu_ref_put(&ca->io_ref); 948 + percpu_ref_put(&ca->io_ref[READ]); 949 949 } 950 950 951 951 static void read_back_super(struct bch_fs *c, struct bch_dev *ca) ··· 963 963 964 964 this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_sb], bio_sectors(bio)); 965 965 966 - percpu_ref_get(&ca->io_ref); 966 + percpu_ref_get(&ca->io_ref[READ]); 967 967 closure_bio_submit(bio, &c->sb_write); 968 968 } 969 969 ··· 989 989 this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_sb], 990 990 bio_sectors(bio)); 991 991 992 - percpu_ref_get(&ca->io_ref); 992 + percpu_ref_get(&ca->io_ref[READ]); 993 993 closure_bio_submit(bio, &c->sb_write); 994 994 } 995 995 ··· 1014 1014 closure_init_stack(cl); 1015 1015 memset(&sb_written, 0, sizeof(sb_written)); 1016 1016 1017 + /* 1018 + * Note: we do writes to RO devices here, and we might want to change 1019 + * that in the future. 1020 + * 1021 + * For now, we expect to be able to call write_super() when we're not 1022 + * yet RW: 1023 + */ 1017 1024 for_each_online_member(c, ca) { 1018 1025 ret = darray_push(&online_devices, ca); 1019 1026 if (bch2_fs_fatal_err_on(ret, c, "%s: error allocating online devices", __func__)) { 1020 - percpu_ref_put(&ca->io_ref); 1027 + percpu_ref_put(&ca->io_ref[READ]); 1021 1028 goto out; 1022 1029 } 1023 - percpu_ref_get(&ca->io_ref); 1030 + percpu_ref_get(&ca->io_ref[READ]); 1024 1031 } 1025 1032 1026 1033 /* Make sure we're using the new magic numbers: */ ··· 1193 1186 /* Make new options visible after they're persistent: */ 1194 1187 bch2_sb_update(c); 1195 1188 darray_for_each(online_devices, ca) 1196 - percpu_ref_put(&(*ca)->io_ref); 1189 + percpu_ref_put(&(*ca)->io_ref[READ]); 1197 1190 darray_exit(&online_devices); 1198 1191 printbuf_exit(&err); 1199 1192 return ret;

+61 -24

fs/bcachefs/super.c

··· 185 185 static void bch2_dev_free(struct bch_dev *); 186 186 static int bch2_dev_alloc(struct bch_fs *, unsigned); 187 187 static int bch2_dev_sysfs_online(struct bch_fs *, struct bch_dev *); 188 + static void bch2_dev_io_ref_stop(struct bch_dev *, int); 188 189 static void __bch2_dev_read_only(struct bch_fs *, struct bch_dev *); 189 190 190 191 struct bch_fs *bch2_dev_to_fs(dev_t dev) ··· 295 294 /* 296 295 * After stopping journal: 297 296 */ 298 - for_each_member_device(c, ca) 297 + for_each_member_device(c, ca) { 298 + bch2_dev_io_ref_stop(ca, WRITE); 299 299 bch2_dev_allocator_remove(c, ca); 300 + } 300 301 } 301 302 302 303 #ifndef BCH_WRITE_REF_DEBUG ··· 468 465 if (ret) 469 466 goto err; 470 467 471 - ret = bch2_fs_mark_dirty(c); 472 - if (ret) 473 - goto err; 474 - 475 468 clear_bit(BCH_FS_clean_shutdown, &c->flags); 476 469 477 470 /* ··· 479 480 set_bit(JOURNAL_need_flush_write, &c->journal.flags); 480 481 set_bit(JOURNAL_running, &c->journal.flags); 481 482 482 - for_each_rw_member(c, ca) 483 + __for_each_online_member(c, ca, BIT(BCH_MEMBER_STATE_rw), READ) { 483 484 bch2_dev_allocator_add(c, ca); 485 + percpu_ref_reinit(&ca->io_ref[WRITE]); 486 + } 484 487 bch2_recalc_capacity(c); 488 + 489 + ret = bch2_fs_mark_dirty(c); 490 + if (ret) 491 + goto err; 492 + 493 + spin_lock(&c->journal.lock); 494 + bch2_journal_space_available(&c->journal); 495 + spin_unlock(&c->journal.lock); 496 + 497 + ret = bch2_journal_reclaim_start(&c->journal); 498 + if (ret) 499 + goto err; 485 500 486 501 set_bit(BCH_FS_rw, &c->flags); 487 502 set_bit(BCH_FS_was_rw, &c->flags); ··· 508 495 atomic_long_inc(&c->writes[i]); 509 496 } 510 497 #endif 511 - 512 - ret = bch2_journal_reclaim_start(&c->journal); 513 - if (ret) 514 - goto err; 515 - 516 498 if (!early) { 517 499 ret = bch2_fs_read_write_late(c); 518 500 if (ret) ··· 683 675 684 676 if (ca) { 685 677 EBUG_ON(atomic_long_read(&ca->ref) != 1); 678 + bch2_dev_io_ref_stop(ca, READ); 686 679 bch2_free_super(&ca->disk_sb); 687 680 bch2_dev_free(ca); 688 681 } ··· 1208 1199 1209 1200 /* Device startup/shutdown: */ 1210 1201 1202 + static void bch2_dev_io_ref_stop(struct bch_dev *ca, int rw) 1203 + { 1204 + if (!percpu_ref_is_zero(&ca->io_ref[rw])) { 1205 + reinit_completion(&ca->io_ref_completion[rw]); 1206 + percpu_ref_kill(&ca->io_ref[rw]); 1207 + wait_for_completion(&ca->io_ref_completion[rw]); 1208 + } 1209 + } 1210 + 1211 1211 static void bch2_dev_release(struct kobject *kobj) 1212 1212 { 1213 1213 struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj); ··· 1226 1208 1227 1209 static void bch2_dev_free(struct bch_dev *ca) 1228 1210 { 1211 + WARN_ON(!percpu_ref_is_zero(&ca->io_ref[WRITE])); 1212 + WARN_ON(!percpu_ref_is_zero(&ca->io_ref[READ])); 1213 + 1229 1214 cancel_work_sync(&ca->io_error_work); 1230 1215 1231 1216 bch2_dev_unlink(ca); ··· 1247 1226 bch2_time_stats_quantiles_exit(&ca->io_latency[WRITE]); 1248 1227 bch2_time_stats_quantiles_exit(&ca->io_latency[READ]); 1249 1228 1250 - percpu_ref_exit(&ca->io_ref); 1229 + percpu_ref_exit(&ca->io_ref[WRITE]); 1230 + percpu_ref_exit(&ca->io_ref[READ]); 1251 1231 #ifndef CONFIG_BCACHEFS_DEBUG 1252 1232 percpu_ref_exit(&ca->ref); 1253 1233 #endif ··· 1260 1238 1261 1239 lockdep_assert_held(&c->state_lock); 1262 1240 1263 - if (percpu_ref_is_zero(&ca->io_ref)) 1241 + if (percpu_ref_is_zero(&ca->io_ref[READ])) 1264 1242 return; 1265 1243 1266 1244 __bch2_dev_read_only(c, ca); 1267 1245 1268 - reinit_completion(&ca->io_ref_completion); 1269 - percpu_ref_kill(&ca->io_ref); 1270 - wait_for_completion(&ca->io_ref_completion); 1246 + bch2_dev_io_ref_stop(ca, READ); 1271 1247 1272 1248 bch2_dev_unlink(ca); 1273 1249 ··· 1282 1262 } 1283 1263 #endif 1284 1264 1285 - static void bch2_dev_io_ref_complete(struct percpu_ref *ref) 1265 + static void bch2_dev_io_ref_read_complete(struct percpu_ref *ref) 1286 1266 { 1287 - struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref); 1267 + struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref[READ]); 1288 1268 1289 - complete(&ca->io_ref_completion); 1269 + complete(&ca->io_ref_completion[READ]); 1270 + } 1271 + 1272 + static void bch2_dev_io_ref_write_complete(struct percpu_ref *ref) 1273 + { 1274 + struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref[WRITE]); 1275 + 1276 + complete(&ca->io_ref_completion[WRITE]); 1290 1277 } 1291 1278 1292 1279 static void bch2_dev_unlink(struct bch_dev *ca) ··· 1357 1330 1358 1331 kobject_init(&ca->kobj, &bch2_dev_ktype); 1359 1332 init_completion(&ca->ref_completion); 1360 - init_completion(&ca->io_ref_completion); 1333 + init_completion(&ca->io_ref_completion[READ]); 1334 + init_completion(&ca->io_ref_completion[WRITE]); 1361 1335 1362 1336 INIT_WORK(&ca->io_error_work, bch2_io_error_work); 1363 1337 ··· 1384 1356 1385 1357 bch2_dev_allocator_background_init(ca); 1386 1358 1387 - if (percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete, 1359 + if (percpu_ref_init(&ca->io_ref[READ], bch2_dev_io_ref_read_complete, 1360 + PERCPU_REF_INIT_DEAD, GFP_KERNEL) || 1361 + percpu_ref_init(&ca->io_ref[WRITE], bch2_dev_io_ref_write_complete, 1388 1362 PERCPU_REF_INIT_DEAD, GFP_KERNEL) || 1389 1363 !(ca->sb_read_scratch = kmalloc(BCH_SB_READ_SCRATCH_BUF_SIZE, GFP_KERNEL)) || 1390 1364 bch2_dev_buckets_alloc(c, ca) || ··· 1449 1419 return -BCH_ERR_device_size_too_small; 1450 1420 } 1451 1421 1452 - BUG_ON(!percpu_ref_is_zero(&ca->io_ref)); 1422 + BUG_ON(!percpu_ref_is_zero(&ca->io_ref[READ])); 1423 + BUG_ON(!percpu_ref_is_zero(&ca->io_ref[WRITE])); 1453 1424 1454 1425 ret = bch2_dev_journal_init(ca, sb->sb); 1455 1426 if (ret) ··· 1469 1438 1470 1439 ca->dev = ca->disk_sb.bdev->bd_dev; 1471 1440 1472 - percpu_ref_reinit(&ca->io_ref); 1441 + percpu_ref_reinit(&ca->io_ref[READ]); 1473 1442 1474 1443 return 0; 1475 1444 } ··· 1599 1568 1600 1569 static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca) 1601 1570 { 1571 + bch2_dev_io_ref_stop(ca, WRITE); 1572 + 1602 1573 /* 1603 1574 * The allocator thread itself allocates btree nodes, so stop it first: 1604 1575 */ ··· 1617 1584 1618 1585 bch2_dev_allocator_add(c, ca); 1619 1586 bch2_recalc_capacity(c); 1587 + 1588 + if (percpu_ref_is_zero(&ca->io_ref[WRITE])) 1589 + percpu_ref_reinit(&ca->io_ref[WRITE]); 1590 + 1620 1591 bch2_dev_do_discards(ca); 1621 1592 } 1622 1593 ··· 1768 1731 return 0; 1769 1732 err: 1770 1733 if (ca->mi.state == BCH_MEMBER_STATE_rw && 1771 - !percpu_ref_is_zero(&ca->io_ref)) 1734 + !percpu_ref_is_zero(&ca->io_ref[READ])) 1772 1735 __bch2_dev_read_write(c, ca); 1773 1736 up_write(&c->state_lock); 1774 1737 return ret;

+15 -15

fs/bcachefs/tests.c

··· 43 43 BTREE_ITER_intent); 44 44 45 45 ret = commit_do(trans, NULL, NULL, 0, 46 - bch2_btree_iter_traverse(&iter) ?: 46 + bch2_btree_iter_traverse(trans, &iter) ?: 47 47 bch2_trans_update(trans, &iter, &k.k_i, 0)); 48 48 bch_err_msg(c, ret, "update error"); 49 49 if (ret) ··· 51 51 52 52 pr_info("deleting once"); 53 53 ret = commit_do(trans, NULL, NULL, 0, 54 - bch2_btree_iter_traverse(&iter) ?: 54 + bch2_btree_iter_traverse(trans, &iter) ?: 55 55 bch2_btree_delete_at(trans, &iter, 0)); 56 56 bch_err_msg(c, ret, "delete error (first)"); 57 57 if (ret) ··· 59 59 60 60 pr_info("deleting twice"); 61 61 ret = commit_do(trans, NULL, NULL, 0, 62 - bch2_btree_iter_traverse(&iter) ?: 62 + bch2_btree_iter_traverse(trans, &iter) ?: 63 63 bch2_btree_delete_at(trans, &iter, 0)); 64 64 bch_err_msg(c, ret, "delete error (second)"); 65 65 if (ret) ··· 84 84 BTREE_ITER_intent); 85 85 86 86 ret = commit_do(trans, NULL, NULL, 0, 87 - bch2_btree_iter_traverse(&iter) ?: 87 + bch2_btree_iter_traverse(trans, &iter) ?: 88 88 bch2_trans_update(trans, &iter, &k.k_i, 0)); 89 89 bch_err_msg(c, ret, "update error"); 90 90 if (ret) ··· 94 94 bch2_journal_flush_all_pins(&c->journal); 95 95 96 96 ret = commit_do(trans, NULL, NULL, 0, 97 - bch2_btree_iter_traverse(&iter) ?: 97 + bch2_btree_iter_traverse(trans, &iter) ?: 98 98 bch2_btree_delete_at(trans, &iter, 0)); 99 99 bch_err_msg(c, ret, "delete error"); 100 100 if (ret) ··· 349 349 bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, 350 350 SPOS(0, 0, U32_MAX), 0); 351 351 352 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)))); 352 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)))); 353 353 BUG_ON(k.k); 354 354 355 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)))); 355 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)))); 356 356 BUG_ON(k.k); 357 357 358 358 bch2_trans_iter_exit(trans, &iter); ··· 369 369 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 370 370 SPOS(0, 0, U32_MAX), 0); 371 371 372 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)))); 372 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)))); 373 373 BUG_ON(k.k); 374 374 375 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)))); 375 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)))); 376 376 BUG_ON(k.k); 377 377 378 378 bch2_trans_iter_exit(trans, &iter); ··· 488 488 trans = bch2_trans_get(c); 489 489 bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, 490 490 SPOS(0, 0, snapid_lo), 0); 491 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)))); 491 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)))); 492 492 493 493 BUG_ON(k.k->p.snapshot != U32_MAX); 494 494 ··· 602 602 SPOS(0, 0, U32_MAX), 0); 603 603 604 604 for (i = 0; i < nr; i++) { 605 - bch2_btree_iter_set_pos(&iter, SPOS(0, test_rand(), U32_MAX)); 605 + bch2_btree_iter_set_pos(trans, &iter, SPOS(0, test_rand(), U32_MAX)); 606 606 607 - lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek(&iter))); 607 + lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek(trans, &iter))); 608 608 ret = bkey_err(k); 609 609 if (ret) 610 610 break; ··· 623 623 struct bkey_s_c k; 624 624 int ret; 625 625 626 - bch2_btree_iter_set_pos(iter, SPOS(0, pos, U32_MAX)); 626 + bch2_btree_iter_set_pos(trans, iter, SPOS(0, pos, U32_MAX)); 627 627 628 - k = bch2_btree_iter_peek(iter); 628 + k = bch2_btree_iter_peek(trans, iter); 629 629 ret = bkey_err(k); 630 630 bch_err_msg(trans->c, ret, "lookup error"); 631 631 if (ret) ··· 672 672 673 673 bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, pos, 674 674 BTREE_ITER_intent); 675 - k = bch2_btree_iter_peek_max(&iter, POS(0, U64_MAX)); 675 + k = bch2_btree_iter_peek_max(trans, &iter, POS(0, U64_MAX)); 676 676 ret = bkey_err(k); 677 677 if (ret) 678 678 goto err;

+1 -1

fs/bcachefs/xattr.c

··· 168 168 int type, int flags) 169 169 { 170 170 struct bch_fs *c = trans->c; 171 - struct btree_iter inode_iter = { NULL }; 171 + struct btree_iter inode_iter = {}; 172 172 int ret; 173 173 174 174 ret = bch2_subvol_is_ro_trans(trans, inum.subvol) ?:

Configure Feed

Configure Feed