bcachefs: remove heap-related macros and switch to generic min_heap

+35 -12

fs/bcachefs/clock.c

··· 6 6 #include <linux/kthread.h> 7 7 #include <linux/preempt.h> 8 8 9 - static inline long io_timer_cmp(io_timer_heap *h, 10 - struct io_timer *l, 11 - struct io_timer *r) 9 + static inline bool io_timer_cmp(const void *l, const void *r, void __always_unused *args) 12 10 { 13 - return l->expire - r->expire; 11 + struct io_timer **_l = (struct io_timer **)l; 12 + struct io_timer **_r = (struct io_timer **)r; 13 + 14 + return (*_l)->expire < (*_r)->expire; 15 + } 16 + 17 + static inline void io_timer_swp(void *l, void *r, void __always_unused *args) 18 + { 19 + struct io_timer **_l = (struct io_timer **)l; 20 + struct io_timer **_r = (struct io_timer **)r; 21 + 22 + swap(*_l, *_r); 14 23 } 15 24 16 25 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer) 17 26 { 18 27 size_t i; 28 + const struct min_heap_callbacks callbacks = { 29 + .less = io_timer_cmp, 30 + .swp = io_timer_swp, 31 + }; 19 32 20 33 spin_lock(&clock->timer_lock); 21 34 ··· 39 26 return; 40 27 } 41 28 42 - for (i = 0; i < clock->timers.used; i++) 29 + for (i = 0; i < clock->timers.nr; i++) 43 30 if (clock->timers.data[i] == timer) 44 31 goto out; 45 32 46 - BUG_ON(!heap_add(&clock->timers, timer, io_timer_cmp, NULL)); 33 + BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL)); 47 34 out: 48 35 spin_unlock(&clock->timer_lock); 49 36 } ··· 51 38 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer) 52 39 { 53 40 size_t i; 41 + const struct min_heap_callbacks callbacks = { 42 + .less = io_timer_cmp, 43 + .swp = io_timer_swp, 44 + }; 54 45 55 46 spin_lock(&clock->timer_lock); 56 47 57 - for (i = 0; i < clock->timers.used; i++) 48 + for (i = 0; i < clock->timers.nr; i++) 58 49 if (clock->timers.data[i] == timer) { 59 - heap_del(&clock->timers, i, io_timer_cmp, NULL); 50 + min_heap_del(&clock->timers, i, &callbacks, NULL); 60 51 break; 61 52 } 62 53 ··· 148 131 unsigned long now) 149 132 { 150 133 struct io_timer *ret = NULL; 134 + const struct min_heap_callbacks callbacks = { 135 + .less = io_timer_cmp, 136 + .swp = io_timer_swp, 137 + }; 151 138 152 139 spin_lock(&clock->timer_lock); 153 140 154 - if (clock->timers.used && 155 - time_after_eq(now, clock->timers.data[0]->expire)) 156 - heap_pop(&clock->timers, ret, io_timer_cmp, NULL); 141 + if (clock->timers.nr && 142 + time_after_eq(now, clock->timers.data[0]->expire)) { 143 + ret = *min_heap_peek(&clock->timers); 144 + min_heap_pop(&clock->timers, &callbacks, NULL); 145 + } 157 146 158 147 spin_unlock(&clock->timer_lock); 159 148 ··· 184 161 spin_lock(&clock->timer_lock); 185 162 now = atomic64_read(&clock->now); 186 163 187 - for (i = 0; i < clock->timers.used; i++) 164 + for (i = 0; i < clock->timers.nr; i++) 188 165 prt_printf(out, "%ps:\t%li\n", 189 166 clock->timers.data[i]->fn, 190 167 clock->timers.data[i]->expire - now);

+1 -1

fs/bcachefs/clock_types.h

··· 23 23 /* Amount to buffer up on a percpu counter */ 24 24 #define IO_CLOCK_PCPU_SECTORS 128 25 25 26 - typedef HEAP(struct io_timer *) io_timer_heap; 26 + typedef DEFINE_MIN_HEAP(struct io_timer *, io_timer_heap) io_timer_heap; 27 27 28 28 struct io_clock { 29 29 atomic64_t now;

+52 -26

fs/bcachefs/ec.c

··· 910 910 911 911 mutex_lock(&c->ec_stripes_heap_lock); 912 912 if (n.size > h->size) { 913 - memcpy(n.data, h->data, h->used * sizeof(h->data[0])); 914 - n.used = h->used; 913 + memcpy(n.data, h->data, h->nr * sizeof(h->data[0])); 914 + n.nr = h->nr; 915 915 swap(*h, n); 916 916 } 917 917 mutex_unlock(&c->ec_stripes_heap_lock); ··· 1002 1002 1003 1003 lockdep_assert_held(&c->ec_stripes_heap_lock); 1004 1004 1005 - if (h->used && 1005 + if (h->nr && 1006 1006 h->data[0].blocks_nonempty == 0 && 1007 1007 !bch2_stripe_is_open(c, h->data[0].idx)) 1008 1008 return h->data[0].idx; 1009 1009 1010 1010 return 0; 1011 - } 1012 - 1013 - static inline int ec_stripes_heap_cmp(ec_stripes_heap *h, 1014 - struct ec_stripe_heap_entry l, 1015 - struct ec_stripe_heap_entry r) 1016 - { 1017 - return ((l.blocks_nonempty > r.blocks_nonempty) - 1018 - (l.blocks_nonempty < r.blocks_nonempty)); 1019 1011 } 1020 1012 1021 1013 static inline void ec_stripes_heap_set_backpointer(ec_stripes_heap *h, ··· 1018 1026 genradix_ptr(&c->stripes, h->data[i].idx)->heap_idx = i; 1019 1027 } 1020 1028 1029 + static inline bool ec_stripes_heap_cmp(const void *l, const void *r, void __always_unused *args) 1030 + { 1031 + struct ec_stripe_heap_entry *_l = (struct ec_stripe_heap_entry *)l; 1032 + struct ec_stripe_heap_entry *_r = (struct ec_stripe_heap_entry *)r; 1033 + 1034 + return ((_l->blocks_nonempty > _r->blocks_nonempty) < 1035 + (_l->blocks_nonempty < _r->blocks_nonempty)); 1036 + } 1037 + 1038 + static inline void ec_stripes_heap_swap(void *l, void *r, void *h) 1039 + { 1040 + struct ec_stripe_heap_entry *_l = (struct ec_stripe_heap_entry *)l; 1041 + struct ec_stripe_heap_entry *_r = (struct ec_stripe_heap_entry *)r; 1042 + ec_stripes_heap *_h = (ec_stripes_heap *)h; 1043 + size_t i = _l - _h->data; 1044 + size_t j = _r - _h->data; 1045 + 1046 + swap(*_l, *_r); 1047 + 1048 + ec_stripes_heap_set_backpointer(_h, i); 1049 + ec_stripes_heap_set_backpointer(_h, j); 1050 + } 1051 + 1021 1052 static void heap_verify_backpointer(struct bch_fs *c, size_t idx) 1022 1053 { 1023 1054 ec_stripes_heap *h = &c->ec_stripes_heap; 1024 1055 struct stripe *m = genradix_ptr(&c->stripes, idx); 1025 1056 1026 - BUG_ON(m->heap_idx >= h->used); 1057 + BUG_ON(m->heap_idx >= h->nr); 1027 1058 BUG_ON(h->data[m->heap_idx].idx != idx); 1028 1059 } 1029 1060 1030 1061 void bch2_stripes_heap_del(struct bch_fs *c, 1031 1062 struct stripe *m, size_t idx) 1032 1063 { 1064 + const struct min_heap_callbacks callbacks = { 1065 + .less = ec_stripes_heap_cmp, 1066 + .swp = ec_stripes_heap_swap, 1067 + }; 1068 + 1033 1069 mutex_lock(&c->ec_stripes_heap_lock); 1034 1070 heap_verify_backpointer(c, idx); 1035 1071 1036 - heap_del(&c->ec_stripes_heap, m->heap_idx, 1037 - ec_stripes_heap_cmp, 1038 - ec_stripes_heap_set_backpointer); 1072 + min_heap_del(&c->ec_stripes_heap, m->heap_idx, &callbacks, &c->ec_stripes_heap); 1039 1073 mutex_unlock(&c->ec_stripes_heap_lock); 1040 1074 } 1041 1075 1042 1076 void bch2_stripes_heap_insert(struct bch_fs *c, 1043 1077 struct stripe *m, size_t idx) 1044 1078 { 1045 - mutex_lock(&c->ec_stripes_heap_lock); 1046 - BUG_ON(heap_full(&c->ec_stripes_heap)); 1079 + const struct min_heap_callbacks callbacks = { 1080 + .less = ec_stripes_heap_cmp, 1081 + .swp = ec_stripes_heap_swap, 1082 + }; 1047 1083 1048 - heap_add(&c->ec_stripes_heap, ((struct ec_stripe_heap_entry) { 1084 + mutex_lock(&c->ec_stripes_heap_lock); 1085 + BUG_ON(min_heap_full(&c->ec_stripes_heap)); 1086 + 1087 + genradix_ptr(&c->stripes, idx)->heap_idx = c->ec_stripes_heap.nr; 1088 + min_heap_push(&c->ec_stripes_heap, &((struct ec_stripe_heap_entry) { 1049 1089 .idx = idx, 1050 1090 .blocks_nonempty = m->blocks_nonempty, 1051 1091 }), 1052 - ec_stripes_heap_cmp, 1053 - ec_stripes_heap_set_backpointer); 1092 + &callbacks, 1093 + &c->ec_stripes_heap); 1054 1094 1055 1095 heap_verify_backpointer(c, idx); 1056 1096 mutex_unlock(&c->ec_stripes_heap_lock); ··· 1091 1067 void bch2_stripes_heap_update(struct bch_fs *c, 1092 1068 struct stripe *m, size_t idx) 1093 1069 { 1070 + const struct min_heap_callbacks callbacks = { 1071 + .less = ec_stripes_heap_cmp, 1072 + .swp = ec_stripes_heap_swap, 1073 + }; 1094 1074 ec_stripes_heap *h = &c->ec_stripes_heap; 1095 1075 bool do_deletes; 1096 1076 size_t i; ··· 1105 1077 h->data[m->heap_idx].blocks_nonempty = m->blocks_nonempty; 1106 1078 1107 1079 i = m->heap_idx; 1108 - heap_sift_up(h, i, ec_stripes_heap_cmp, 1109 - ec_stripes_heap_set_backpointer); 1110 - heap_sift_down(h, i, ec_stripes_heap_cmp, 1111 - ec_stripes_heap_set_backpointer); 1080 + min_heap_sift_up(h, i, &callbacks, &c->ec_stripes_heap); 1081 + min_heap_sift_down(h, i, &callbacks, &c->ec_stripes_heap); 1112 1082 1113 1083 heap_verify_backpointer(c, idx); 1114 1084 ··· 1899 1873 return -1; 1900 1874 1901 1875 mutex_lock(&c->ec_stripes_heap_lock); 1902 - for (heap_idx = 0; heap_idx < h->used; heap_idx++) { 1876 + for (heap_idx = 0; heap_idx < h->nr; heap_idx++) { 1903 1877 /* No blocks worth reusing, stripe will just be deleted: */ 1904 1878 if (!h->data[heap_idx].blocks_nonempty) 1905 1879 continue; ··· 2230 2204 size_t i; 2231 2205 2232 2206 mutex_lock(&c->ec_stripes_heap_lock); 2233 - for (i = 0; i < min_t(size_t, h->used, 50); i++) { 2207 + for (i = 0; i < min_t(size_t, h->nr, 50); i++) { 2234 2208 m = genradix_ptr(&c->stripes, h->data[i].idx); 2235 2209 2236 2210 prt_printf(out, "%zu %u/%u+%u", h->data[i].idx,

+1 -1

fs/bcachefs/ec_types.h

··· 36 36 unsigned blocks_nonempty; 37 37 }; 38 38 39 - typedef HEAP(struct ec_stripe_heap_entry) ec_stripes_heap; 39 + typedef DEFINE_MIN_HEAP(struct ec_stripe_heap_entry, ec_stripes_heap) ec_stripes_heap; 40 40 41 41 #endif /* _BCACHEFS_EC_TYPES_H */

+2 -116

fs/bcachefs/util.h

··· 8 8 #include <linux/errno.h> 9 9 #include <linux/freezer.h> 10 10 #include <linux/kernel.h> 11 + #include <linux/min_heap.h> 11 12 #include <linux/sched/clock.h> 12 13 #include <linux/llist.h> 13 14 #include <linux/log2.h> ··· 55 54 PAGE_SIZE); 56 55 } 57 56 58 - #define HEAP(type) \ 59 - struct { \ 60 - size_t size, used; \ 61 - type *data; \ 62 - } 63 - 64 - #define DECLARE_HEAP(type, name) HEAP(type) name 65 - 66 57 #define init_heap(heap, _size, gfp) \ 67 58 ({ \ 68 - (heap)->used = 0; \ 59 + (heap)->nr = 0; \ 69 60 (heap)->size = (_size); \ 70 61 (heap)->data = kvmalloc((heap)->size * sizeof((heap)->data[0]),\ 71 62 (gfp)); \ ··· 67 74 do { \ 68 75 kvfree((heap)->data); \ 69 76 (heap)->data = NULL; \ 70 - } while (0) 71 - 72 - #define heap_set_backpointer(h, i, _fn) \ 73 - do { \ 74 - void (*fn)(typeof(h), size_t) = _fn; \ 75 - if (fn) \ 76 - fn(h, i); \ 77 - } while (0) 78 - 79 - #define heap_swap(h, i, j, set_backpointer) \ 80 - do { \ 81 - swap((h)->data[i], (h)->data[j]); \ 82 - heap_set_backpointer(h, i, set_backpointer); \ 83 - heap_set_backpointer(h, j, set_backpointer); \ 84 - } while (0) 85 - 86 - #define heap_peek(h) \ 87 - ({ \ 88 - EBUG_ON(!(h)->used); \ 89 - (h)->data[0]; \ 90 - }) 91 - 92 - #define heap_full(h) ((h)->used == (h)->size) 93 - 94 - #define heap_sift_down(h, i, cmp, set_backpointer) \ 95 - do { \ 96 - size_t _c, _j = i; \ 97 - \ 98 - for (; _j * 2 + 1 < (h)->used; _j = _c) { \ 99 - _c = _j * 2 + 1; \ 100 - if (_c + 1 < (h)->used && \ 101 - cmp(h, (h)->data[_c], (h)->data[_c + 1]) >= 0) \ 102 - _c++; \ 103 - \ 104 - if (cmp(h, (h)->data[_c], (h)->data[_j]) >= 0) \ 105 - break; \ 106 - heap_swap(h, _c, _j, set_backpointer); \ 107 - } \ 108 - } while (0) 109 - 110 - #define heap_sift_up(h, i, cmp, set_backpointer) \ 111 - do { \ 112 - while (i) { \ 113 - size_t p = (i - 1) / 2; \ 114 - if (cmp(h, (h)->data[i], (h)->data[p]) >= 0) \ 115 - break; \ 116 - heap_swap(h, i, p, set_backpointer); \ 117 - i = p; \ 118 - } \ 119 - } while (0) 120 - 121 - #define __heap_add(h, d, cmp, set_backpointer) \ 122 - ({ \ 123 - size_t _i = (h)->used++; \ 124 - (h)->data[_i] = d; \ 125 - heap_set_backpointer(h, _i, set_backpointer); \ 126 - \ 127 - heap_sift_up(h, _i, cmp, set_backpointer); \ 128 - _i; \ 129 - }) 130 - 131 - #define heap_add(h, d, cmp, set_backpointer) \ 132 - ({ \ 133 - bool _r = !heap_full(h); \ 134 - if (_r) \ 135 - __heap_add(h, d, cmp, set_backpointer); \ 136 - _r; \ 137 - }) 138 - 139 - #define heap_add_or_replace(h, new, cmp, set_backpointer) \ 140 - do { \ 141 - if (!heap_add(h, new, cmp, set_backpointer) && \ 142 - cmp(h, new, heap_peek(h)) >= 0) { \ 143 - (h)->data[0] = new; \ 144 - heap_set_backpointer(h, 0, set_backpointer); \ 145 - heap_sift_down(h, 0, cmp, set_backpointer); \ 146 - } \ 147 - } while (0) 148 - 149 - #define heap_del(h, i, cmp, set_backpointer) \ 150 - do { \ 151 - size_t _i = (i); \ 152 - \ 153 - BUG_ON(_i >= (h)->used); \ 154 - (h)->used--; \ 155 - if ((_i) < (h)->used) { \ 156 - heap_swap(h, _i, (h)->used, set_backpointer); \ 157 - heap_sift_up(h, _i, cmp, set_backpointer); \ 158 - heap_sift_down(h, _i, cmp, set_backpointer); \ 159 - } \ 160 - } while (0) 161 - 162 - #define heap_pop(h, d, cmp, set_backpointer) \ 163 - ({ \ 164 - bool _r = (h)->used; \ 165 - if (_r) { \ 166 - (d) = (h)->data[0]; \ 167 - heap_del(h, 0, cmp, set_backpointer); \ 168 - } \ 169 - _r; \ 170 - }) 171 - 172 - #define heap_resort(heap, cmp, set_backpointer) \ 173 - do { \ 174 - ssize_t _i; \ 175 - for (_i = (ssize_t) (heap)->used / 2 - 1; _i >= 0; --_i) \ 176 - heap_sift_down(heap, _i, cmp, set_backpointer); \ 177 77 } while (0) 178 78 179 79 #define ANYSINT_MAX(t) \

Configure Feed

Configure Feed