Merge tag 'bitmap-6.1-rc1' of https://github.com/norov/linux

+1 -1

arch/loongarch/kernel/setup.c

··· 336 336 for (; i < NR_CPUS; i++) 337 337 set_cpu_possible(i, false); 338 338 339 - nr_cpu_ids = possible; 339 + set_nr_cpu_ids(possible); 340 340 } 341 341 #endif 342 342

+1 -1

arch/mips/kernel/setup.c

··· 751 751 for (; i < NR_CPUS; i++) 752 752 set_cpu_possible(i, false); 753 753 754 - nr_cpu_ids = possible; 754 + set_nr_cpu_ids(possible); 755 755 } 756 756 #else 757 757 static inline void prefill_possible_map(void) {}

+4

arch/powerpc/kernel/head_64.S

··· 393 393 #else 394 394 LOAD_REG_ADDR(r8, paca_ptrs) /* Load paca_ptrs pointe */ 395 395 ld r8,0(r8) /* Get base vaddr of array */ 396 + #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 397 + LOAD_REG_IMMEDIATE(r7, NR_CPUS) 398 + #else 396 399 LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */ 397 400 lwz r7,0(r7) /* also the max paca allocated */ 401 + #endif 398 402 li r5,0 /* logical cpu id */ 399 403 1: 400 404 sldi r9,r5,3 /* get paca_ptrs[] index from cpu id */

+2 -2

arch/x86/kernel/smpboot.c

··· 1316 1316 nr++; 1317 1317 } 1318 1318 1319 - nr_cpu_ids = 8; 1319 + set_nr_cpu_ids(8); 1320 1320 } 1321 1321 #endif 1322 1322 ··· 1569 1569 possible = i; 1570 1570 } 1571 1571 1572 - nr_cpu_ids = possible; 1572 + set_nr_cpu_ids(possible); 1573 1573 1574 1574 pr_info("Allowing %d CPUs, %d hotplug CPUs\n", 1575 1575 possible, max_t(int, possible - num_processors, 0));

+1 -1

arch/x86/xen/smp_pv.c

··· 179 179 * hypercall to expand the max number of VCPUs an already 180 180 * running guest has. So cap it up to X. */ 181 181 if (subtract) 182 - nr_cpu_ids = nr_cpu_ids - subtract; 182 + set_nr_cpu_ids(nr_cpu_ids - subtract); 183 183 #endif 184 184 185 185 }

+2 -2

fs/ntfs3/bitmap.c

··· 560 560 561 561 buf = (ulong *)bh->b_data; 562 562 563 - used = __bitmap_weight(buf, wbits); 563 + used = bitmap_weight(buf, wbits); 564 564 if (used < wbits) { 565 565 frb = wbits - used; 566 566 wnd->free_bits[iw] = frb; ··· 1364 1364 buf = (ulong *)bh->b_data; 1365 1365 1366 1366 __bitmap_clear(buf, b0, blocksize * 8 - b0); 1367 - frb = wbits - __bitmap_weight(buf, wbits); 1367 + frb = wbits - bitmap_weight(buf, wbits); 1368 1368 wnd->total_zeroes += frb - wnd->free_bits[iw]; 1369 1369 wnd->free_bits[iw] = frb; 1370 1370

+12 -1

include/linux/bitmap.h

··· 51 51 * bitmap_empty(src, nbits) Are all bits zero in *src? 52 52 * bitmap_full(src, nbits) Are all bits set in *src? 53 53 * bitmap_weight(src, nbits) Hamming Weight: number set bits 54 + * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap 54 55 * bitmap_set(dst, pos, nbits) Set specified bit area 55 56 * bitmap_clear(dst, pos, nbits) Clear specified bit area 56 57 * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area ··· 165 164 bool __bitmap_subset(const unsigned long *bitmap1, 166 165 const unsigned long *bitmap2, unsigned int nbits); 167 166 unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); 167 + unsigned int __bitmap_weight_and(const unsigned long *bitmap1, 168 + const unsigned long *bitmap2, unsigned int nbits); 168 169 void __bitmap_set(unsigned long *map, unsigned int start, int len); 169 170 void __bitmap_clear(unsigned long *map, unsigned int start, int len); 170 171 ··· 225 222 #else 226 223 #define bitmap_copy_le bitmap_copy 227 224 #endif 228 - unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits); 229 225 int bitmap_print_to_pagebuf(bool list, char *buf, 230 226 const unsigned long *maskp, int nmaskbits); 231 227 ··· 439 437 if (small_const_nbits(nbits)) 440 438 return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); 441 439 return __bitmap_weight(src, nbits); 440 + } 441 + 442 + static __always_inline 443 + unsigned long bitmap_weight_and(const unsigned long *src1, 444 + const unsigned long *src2, unsigned int nbits) 445 + { 446 + if (small_const_nbits(nbits)) 447 + return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)); 448 + return __bitmap_weight_and(src1, src2, nbits); 442 449 } 443 450 444 451 static __always_inline void bitmap_set(unsigned long *map, unsigned int start,

+19

include/linux/bitops.h

··· 248 248 } 249 249 250 250 /** 251 + * fns - find N'th set bit in a word 252 + * @word: The word to search 253 + * @n: Bit to find 254 + */ 255 + static inline unsigned long fns(unsigned long word, unsigned int n) 256 + { 257 + unsigned int bit; 258 + 259 + while (word) { 260 + bit = __ffs(word); 261 + if (n-- == 0) 262 + return bit; 263 + __clear_bit(bit, &word); 264 + } 265 + 266 + return BITS_PER_LONG; 267 + } 268 + 269 + /** 251 270 * assign_bit - Assign value to a bit in memory 252 271 * @nr: the bit to set 253 272 * @addr: the address to start counting from

+97 -35

include/linux/cpumask.h

··· 35 35 */ 36 36 #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) 37 37 38 - #if NR_CPUS == 1 39 - #define nr_cpu_ids 1U 38 + #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 39 + #define nr_cpu_ids ((unsigned int)NR_CPUS) 40 40 #else 41 41 extern unsigned int nr_cpu_ids; 42 42 #endif 43 43 44 - #ifdef CONFIG_CPUMASK_OFFSTACK 45 - /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also, 46 - * not all bits may be allocated. */ 47 - #define nr_cpumask_bits nr_cpu_ids 44 + static inline void set_nr_cpu_ids(unsigned int nr) 45 + { 46 + #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 47 + WARN_ON(nr != nr_cpu_ids); 48 48 #else 49 - #define nr_cpumask_bits ((unsigned int)NR_CPUS) 49 + nr_cpu_ids = nr; 50 50 #endif 51 + } 52 + 53 + /* Deprecated. Always use nr_cpu_ids. */ 54 + #define nr_cpumask_bits nr_cpu_ids 51 55 52 56 /* 53 57 * The following particular system cpumasks and operations manage ··· 70 66 * representing which CPUs are currently plugged in. And 71 67 * cpu_online_mask is the dynamic subset of cpu_present_mask, 72 68 * indicating those CPUs available for scheduling. 73 - * 74 - * If HOTPLUG is enabled, then cpu_possible_mask is forced to have 75 - * all NR_CPUS bits set, otherwise it is just the set of CPUs that 76 - * ACPI reports present at boot. 77 69 * 78 70 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, 79 71 * depending on what ACPI reports as currently plugged in, otherwise ··· 174 174 static inline 175 175 unsigned int cpumask_next(int n, const struct cpumask *srcp) 176 176 { 177 - /* -1 is a legal arg here. */ 178 - if (n != -1) 179 - cpumask_check(n); 177 + /* n is a prior cpu */ 178 + cpumask_check(n + 1); 180 179 return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); 181 180 } 182 181 ··· 188 189 */ 189 190 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 190 191 { 191 - /* -1 is a legal arg here. */ 192 - if (n != -1) 193 - cpumask_check(n); 192 + /* n is a prior cpu */ 193 + cpumask_check(n + 1); 194 194 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); 195 195 } 196 196 ··· 229 231 unsigned int cpumask_next_and(int n, const struct cpumask *src1p, 230 232 const struct cpumask *src2p) 231 233 { 232 - /* -1 is a legal arg here. */ 233 - if (n != -1) 234 - cpumask_check(n); 234 + /* n is a prior cpu */ 235 + cpumask_check(n + 1); 235 236 return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), 236 237 nr_cpumask_bits, n + 1); 237 238 } ··· 243 246 * After the loop, cpu is >= nr_cpu_ids. 244 247 */ 245 248 #define for_each_cpu(cpu, mask) \ 246 - for ((cpu) = -1; \ 247 - (cpu) = cpumask_next((cpu), (mask)), \ 248 - (cpu) < nr_cpu_ids;) 249 + for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) 249 250 250 251 /** 251 252 * for_each_cpu_not - iterate over every cpu in a complemented mask ··· 253 258 * After the loop, cpu is >= nr_cpu_ids. 254 259 */ 255 260 #define for_each_cpu_not(cpu, mask) \ 256 - for ((cpu) = -1; \ 257 - (cpu) = cpumask_next_zero((cpu), (mask)), \ 258 - (cpu) < nr_cpu_ids;) 261 + for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) 259 262 260 263 #if NR_CPUS == 1 261 264 static inline 262 265 unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) 263 266 { 264 267 cpumask_check(start); 265 - if (n != -1) 266 - cpumask_check(n); 268 + /* n is a prior cpu */ 269 + cpumask_check(n + 1); 267 270 268 271 /* 269 272 * Return the first available CPU when wrapping, or when starting before cpu0, ··· 286 293 * 287 294 * After the loop, cpu is >= nr_cpu_ids. 288 295 */ 289 - #define for_each_cpu_wrap(cpu, mask, start) \ 290 - for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \ 291 - (cpu) < nr_cpumask_bits; \ 292 - (cpu) = cpumask_next_wrap((cpu), (mask), (start), true)) 296 + #define for_each_cpu_wrap(cpu, mask, start) \ 297 + for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start) 293 298 294 299 /** 295 300 * for_each_cpu_and - iterate over every cpu in both masks ··· 304 313 * After the loop, cpu is >= nr_cpu_ids. 305 314 */ 306 315 #define for_each_cpu_and(cpu, mask1, mask2) \ 307 - for ((cpu) = -1; \ 308 - (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ 309 - (cpu) < nr_cpu_ids;) 316 + for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) 317 + 318 + /** 319 + * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding 320 + * those present in another. 321 + * @cpu: the (optionally unsigned) integer iterator 322 + * @mask1: the first cpumask pointer 323 + * @mask2: the second cpumask pointer 324 + * 325 + * This saves a temporary CPU mask in many places. It is equivalent to: 326 + * struct cpumask tmp; 327 + * cpumask_andnot(&tmp, &mask1, &mask2); 328 + * for_each_cpu(cpu, &tmp) 329 + * ... 330 + * 331 + * After the loop, cpu is >= nr_cpu_ids. 332 + */ 333 + #define for_each_cpu_andnot(cpu, mask1, mask2) \ 334 + for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) 310 335 311 336 /** 312 337 * cpumask_any_but - return a "random" in a cpumask, but not this one. ··· 342 335 if (i != cpu) 343 336 break; 344 337 return i; 338 + } 339 + 340 + /** 341 + * cpumask_nth - get the first cpu in a cpumask 342 + * @srcp: the cpumask pointer 343 + * @cpu: the N'th cpu to find, starting from 0 344 + * 345 + * Returns >= nr_cpu_ids if such cpu doesn't exist. 346 + */ 347 + static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) 348 + { 349 + return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu)); 350 + } 351 + 352 + /** 353 + * cpumask_nth_and - get the first cpu in 2 cpumasks 354 + * @srcp1: the cpumask pointer 355 + * @srcp2: the cpumask pointer 356 + * @cpu: the N'th cpu to find, starting from 0 357 + * 358 + * Returns >= nr_cpu_ids if such cpu doesn't exist. 359 + */ 360 + static inline 361 + unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, 362 + const struct cpumask *srcp2) 363 + { 364 + return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 365 + nr_cpumask_bits, cpumask_check(cpu)); 366 + } 367 + 368 + /** 369 + * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. 370 + * @srcp1: the cpumask pointer 371 + * @srcp2: the cpumask pointer 372 + * @cpu: the N'th cpu to find, starting from 0 373 + * 374 + * Returns >= nr_cpu_ids if such cpu doesn't exist. 375 + */ 376 + static inline 377 + unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, 378 + const struct cpumask *srcp2) 379 + { 380 + return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 381 + nr_cpumask_bits, cpumask_check(cpu)); 345 382 } 346 383 347 384 #define CPU_BITS_NONE \ ··· 635 584 static inline unsigned int cpumask_weight(const struct cpumask *srcp) 636 585 { 637 586 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); 587 + } 588 + 589 + /** 590 + * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) 591 + * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 592 + * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 593 + */ 594 + static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, 595 + const struct cpumask *srcp2) 596 + { 597 + return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); 638 598 } 639 599 640 600 /**

+269 -41

include/linux/find.h

··· 8 8 9 9 #include <linux/bitops.h> 10 10 11 - extern unsigned long _find_next_bit(const unsigned long *addr1, 12 - const unsigned long *addr2, unsigned long nbits, 13 - unsigned long start, unsigned long invert, unsigned long le); 11 + unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, 12 + unsigned long start); 13 + unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, 14 + unsigned long nbits, unsigned long start); 15 + unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, 16 + unsigned long nbits, unsigned long start); 17 + unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, 18 + unsigned long start); 14 19 extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); 20 + unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n); 21 + unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, 22 + unsigned long size, unsigned long n); 23 + unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, 24 + unsigned long size, unsigned long n); 15 25 extern unsigned long _find_first_and_bit(const unsigned long *addr1, 16 26 const unsigned long *addr2, unsigned long size); 17 27 extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); 18 28 extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); 29 + 30 + #ifdef __BIG_ENDIAN 31 + unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size); 32 + unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned 33 + long size, unsigned long offset); 34 + unsigned long _find_next_bit_le(const unsigned long *addr, unsigned 35 + long size, unsigned long offset); 36 + #endif 19 37 20 38 #ifndef find_next_bit 21 39 /** ··· 59 41 return val ? __ffs(val) : size; 60 42 } 61 43 62 - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); 44 + return _find_next_bit(addr, size, offset); 63 45 } 64 46 #endif 65 47 ··· 89 71 return val ? __ffs(val) : size; 90 72 } 91 73 92 - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); 74 + return _find_next_and_bit(addr1, addr2, size, offset); 75 + } 76 + #endif 77 + 78 + #ifndef find_next_andnot_bit 79 + /** 80 + * find_next_andnot_bit - find the next set bit in *addr1 excluding all the bits 81 + * in *addr2 82 + * @addr1: The first address to base the search on 83 + * @addr2: The second address to base the search on 84 + * @size: The bitmap size in bits 85 + * @offset: The bitnumber to start searching at 86 + * 87 + * Returns the bit number for the next set bit 88 + * If no bits are set, returns @size. 89 + */ 90 + static inline 91 + unsigned long find_next_andnot_bit(const unsigned long *addr1, 92 + const unsigned long *addr2, unsigned long size, 93 + unsigned long offset) 94 + { 95 + if (small_const_nbits(size)) { 96 + unsigned long val; 97 + 98 + if (unlikely(offset >= size)) 99 + return size; 100 + 101 + val = *addr1 & ~*addr2 & GENMASK(size - 1, offset); 102 + return val ? __ffs(val) : size; 103 + } 104 + 105 + return _find_next_andnot_bit(addr1, addr2, size, offset); 93 106 } 94 107 #endif 95 108 ··· 148 99 return val == ~0UL ? size : ffz(val); 149 100 } 150 101 151 - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); 102 + return _find_next_zero_bit(addr, size, offset); 152 103 } 153 104 #endif 154 105 ··· 173 124 return _find_first_bit(addr, size); 174 125 } 175 126 #endif 127 + 128 + /** 129 + * find_nth_bit - find N'th set bit in a memory region 130 + * @addr: The address to start the search at 131 + * @size: The maximum number of bits to search 132 + * @n: The number of set bit, which position is needed, counting from 0 133 + * 134 + * The following is semantically equivalent: 135 + * idx = find_nth_bit(addr, size, 0); 136 + * idx = find_first_bit(addr, size); 137 + * 138 + * Returns the bit number of the N'th set bit. 139 + * If no such, returns @size. 140 + */ 141 + static inline 142 + unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) 143 + { 144 + if (n >= size) 145 + return size; 146 + 147 + if (small_const_nbits(size)) { 148 + unsigned long val = *addr & GENMASK(size - 1, 0); 149 + 150 + return val ? fns(val, n) : size; 151 + } 152 + 153 + return __find_nth_bit(addr, size, n); 154 + } 155 + 156 + /** 157 + * find_nth_and_bit - find N'th set bit in 2 memory regions 158 + * @addr1: The 1st address to start the search at 159 + * @addr2: The 2nd address to start the search at 160 + * @size: The maximum number of bits to search 161 + * @n: The number of set bit, which position is needed, counting from 0 162 + * 163 + * Returns the bit number of the N'th set bit. 164 + * If no such, returns @size. 165 + */ 166 + static inline 167 + unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, 168 + unsigned long size, unsigned long n) 169 + { 170 + if (n >= size) 171 + return size; 172 + 173 + if (small_const_nbits(size)) { 174 + unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); 175 + 176 + return val ? fns(val, n) : size; 177 + } 178 + 179 + return __find_nth_and_bit(addr1, addr2, size, n); 180 + } 181 + 182 + /** 183 + * find_nth_andnot_bit - find N'th set bit in 2 memory regions, 184 + * flipping bits in 2nd region 185 + * @addr1: The 1st address to start the search at 186 + * @addr2: The 2nd address to start the search at 187 + * @size: The maximum number of bits to search 188 + * @n: The number of set bit, which position is needed, counting from 0 189 + * 190 + * Returns the bit number of the N'th set bit. 191 + * If no such, returns @size. 192 + */ 193 + static inline 194 + unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, 195 + unsigned long size, unsigned long n) 196 + { 197 + if (n >= size) 198 + return size; 199 + 200 + if (small_const_nbits(size)) { 201 + unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0); 202 + 203 + return val ? fns(val, n) : size; 204 + } 205 + 206 + return __find_nth_andnot_bit(addr1, addr2, size, n); 207 + } 176 208 177 209 #ifndef find_first_and_bit 178 210 /** ··· 324 194 #endif 325 195 326 196 /** 197 + * find_next_and_bit_wrap - find the next set bit in both memory regions 198 + * @addr1: The first address to base the search on 199 + * @addr2: The second address to base the search on 200 + * @size: The bitmap size in bits 201 + * @offset: The bitnumber to start searching at 202 + * 203 + * Returns the bit number for the next set bit, or first set bit up to @offset 204 + * If no bits are set, returns @size. 205 + */ 206 + static inline 207 + unsigned long find_next_and_bit_wrap(const unsigned long *addr1, 208 + const unsigned long *addr2, 209 + unsigned long size, unsigned long offset) 210 + { 211 + unsigned long bit = find_next_and_bit(addr1, addr2, size, offset); 212 + 213 + if (bit < size) 214 + return bit; 215 + 216 + bit = find_first_and_bit(addr1, addr2, offset); 217 + return bit < offset ? bit : size; 218 + } 219 + 220 + /** 221 + * find_next_bit_wrap - find the next set bit in both memory regions 222 + * @addr: The first address to base the search on 223 + * @size: The bitmap size in bits 224 + * @offset: The bitnumber to start searching at 225 + * 226 + * Returns the bit number for the next set bit, or first set bit up to @offset 227 + * If no bits are set, returns @size. 228 + */ 229 + static inline 230 + unsigned long find_next_bit_wrap(const unsigned long *addr, 231 + unsigned long size, unsigned long offset) 232 + { 233 + unsigned long bit = find_next_bit(addr, size, offset); 234 + 235 + if (bit < size) 236 + return bit; 237 + 238 + bit = find_first_bit(addr, offset); 239 + return bit < offset ? bit : size; 240 + } 241 + 242 + /* 243 + * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing 244 + * before using it alone. 245 + */ 246 + static inline 247 + unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size, 248 + unsigned long start, unsigned long n) 249 + { 250 + unsigned long bit; 251 + 252 + /* If not wrapped around */ 253 + if (n > start) { 254 + /* and have a bit, just return it. */ 255 + bit = find_next_bit(bitmap, size, n); 256 + if (bit < size) 257 + return bit; 258 + 259 + /* Otherwise, wrap around and ... */ 260 + n = 0; 261 + } 262 + 263 + /* Search the other part. */ 264 + bit = find_next_bit(bitmap, start, n); 265 + return bit < start ? bit : size; 266 + } 267 + 268 + /** 327 269 * find_next_clump8 - find next 8-bit clump with set bits in a memory region 328 270 * @clump: location to store copy of found clump 329 271 * @addr: address to base the search on ··· 449 247 return val == ~0UL ? size : ffz(val); 450 248 } 451 249 452 - return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); 250 + return _find_next_zero_bit_le(addr, size, offset); 251 + } 252 + #endif 253 + 254 + #ifndef find_first_zero_bit_le 255 + static inline 256 + unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) 257 + { 258 + if (small_const_nbits(size)) { 259 + unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0); 260 + 261 + return val == ~0UL ? size : ffz(val); 262 + } 263 + 264 + return _find_first_zero_bit_le(addr, size); 453 265 } 454 266 #endif 455 267 ··· 482 266 return val ? __ffs(val) : size; 483 267 } 484 268 485 - return _find_next_bit(addr, NULL, size, offset, 0UL, 1); 269 + return _find_next_bit_le(addr, size, offset); 486 270 } 487 - #endif 488 - 489 - #ifndef find_first_zero_bit_le 490 - #define find_first_zero_bit_le(addr, size) \ 491 - find_next_zero_bit_le((addr), (size), 0) 492 271 #endif 493 272 494 273 #else ··· 491 280 #endif 492 281 493 282 #define for_each_set_bit(bit, addr, size) \ 494 - for ((bit) = find_next_bit((addr), (size), 0); \ 495 - (bit) < (size); \ 496 - (bit) = find_next_bit((addr), (size), (bit) + 1)) 283 + for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) 284 + 285 + #define for_each_and_bit(bit, addr1, addr2, size) \ 286 + for ((bit) = 0; \ 287 + (bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ 288 + (bit)++) 289 + 290 + #define for_each_andnot_bit(bit, addr1, addr2, size) \ 291 + for ((bit) = 0; \ 292 + (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ 293 + (bit)++) 497 294 498 295 /* same as for_each_set_bit() but use bit as value to start with */ 499 296 #define for_each_set_bit_from(bit, addr, size) \ 500 - for ((bit) = find_next_bit((addr), (size), (bit)); \ 501 - (bit) < (size); \ 502 - (bit) = find_next_bit((addr), (size), (bit) + 1)) 297 + for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) 503 298 504 299 #define for_each_clear_bit(bit, addr, size) \ 505 - for ((bit) = find_next_zero_bit((addr), (size), 0); \ 506 - (bit) < (size); \ 507 - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) 300 + for ((bit) = 0; \ 301 + (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \ 302 + (bit)++) 508 303 509 304 /* same as for_each_clear_bit() but use bit as value to start with */ 510 305 #define for_each_clear_bit_from(bit, addr, size) \ 511 - for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ 512 - (bit) < (size); \ 513 - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) 306 + for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++) 514 307 515 308 /** 516 309 * for_each_set_bitrange - iterate over all set bit ranges [b; e) ··· 524 309 * @size: bitmap size in number of bits 525 310 */ 526 311 #define for_each_set_bitrange(b, e, addr, size) \ 527 - for ((b) = find_next_bit((addr), (size), 0), \ 528 - (e) = find_next_zero_bit((addr), (size), (b) + 1); \ 312 + for ((b) = 0; \ 313 + (b) = find_next_bit((addr), (size), b), \ 314 + (e) = find_next_zero_bit((addr), (size), (b) + 1), \ 529 315 (b) < (size); \ 530 - (b) = find_next_bit((addr), (size), (e) + 1), \ 531 - (e) = find_next_zero_bit((addr), (size), (b) + 1)) 316 + (b) = (e) + 1) 532 317 533 318 /** 534 319 * for_each_set_bitrange_from - iterate over all set bit ranges [b; e) ··· 538 323 * @size: bitmap size in number of bits 539 324 */ 540 325 #define for_each_set_bitrange_from(b, e, addr, size) \ 541 - for ((b) = find_next_bit((addr), (size), (b)), \ 542 - (e) = find_next_zero_bit((addr), (size), (b) + 1); \ 326 + for (; \ 327 + (b) = find_next_bit((addr), (size), (b)), \ 328 + (e) = find_next_zero_bit((addr), (size), (b) + 1), \ 543 329 (b) < (size); \ 544 - (b) = find_next_bit((addr), (size), (e) + 1), \ 545 - (e) = find_next_zero_bit((addr), (size), (b) + 1)) 330 + (b) = (e) + 1) 546 331 547 332 /** 548 333 * for_each_clear_bitrange - iterate over all unset bit ranges [b; e) ··· 552 337 * @size: bitmap size in number of bits 553 338 */ 554 339 #define for_each_clear_bitrange(b, e, addr, size) \ 555 - for ((b) = find_next_zero_bit((addr), (size), 0), \ 556 - (e) = find_next_bit((addr), (size), (b) + 1); \ 340 + for ((b) = 0; \ 341 + (b) = find_next_zero_bit((addr), (size), (b)), \ 342 + (e) = find_next_bit((addr), (size), (b) + 1), \ 557 343 (b) < (size); \ 558 - (b) = find_next_zero_bit((addr), (size), (e) + 1), \ 559 - (e) = find_next_bit((addr), (size), (b) + 1)) 344 + (b) = (e) + 1) 560 345 561 346 /** 562 347 * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) ··· 566 351 * @size: bitmap size in number of bits 567 352 */ 568 353 #define for_each_clear_bitrange_from(b, e, addr, size) \ 569 - for ((b) = find_next_zero_bit((addr), (size), (b)), \ 570 - (e) = find_next_bit((addr), (size), (b) + 1); \ 354 + for (; \ 355 + (b) = find_next_zero_bit((addr), (size), (b)), \ 356 + (e) = find_next_bit((addr), (size), (b) + 1), \ 571 357 (b) < (size); \ 572 - (b) = find_next_zero_bit((addr), (size), (e) + 1), \ 573 - (e) = find_next_bit((addr), (size), (b) + 1)) 358 + (b) = (e) + 1) 359 + 360 + /** 361 + * for_each_set_bit_wrap - iterate over all set bits starting from @start, and 362 + * wrapping around the end of bitmap. 363 + * @bit: offset for current iteration 364 + * @addr: bitmap address to base the search on 365 + * @size: bitmap size in number of bits 366 + * @start: Starting bit for bitmap traversing, wrapping around the bitmap end 367 + */ 368 + #define for_each_set_bit_wrap(bit, addr, size, start) \ 369 + for ((bit) = find_next_bit_wrap((addr), (size), (start)); \ 370 + (bit) < (size); \ 371 + (bit) = __for_each_wrap((addr), (size), (start), (bit) + 1)) 574 372 575 373 /** 576 374 * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits

+4 -6

include/linux/netdevice.h

··· 3663 3663 static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp, 3664 3664 unsigned int nr_bits) 3665 3665 { 3666 - /* -1 is a legal arg here. */ 3667 - if (n != -1) 3668 - cpu_max_bits_warn(n, nr_bits); 3666 + /* n is a prior cpu */ 3667 + cpu_max_bits_warn(n + 1, nr_bits); 3669 3668 3670 3669 if (srcp) 3671 3670 return find_next_bit(srcp, nr_bits, n + 1); ··· 3685 3686 const unsigned long *src2p, 3686 3687 unsigned int nr_bits) 3687 3688 { 3688 - /* -1 is a legal arg here. */ 3689 - if (n != -1) 3690 - cpu_max_bits_warn(n, nr_bits); 3689 + /* n is a prior cpu */ 3690 + cpu_max_bits_warn(n + 1, nr_bits); 3691 3691 3692 3692 if (src1p && src2p) 3693 3693 return find_next_and_bit(src1p, src2p, nr_bits, n + 1);

+1 -2

include/linux/nodemask.h

··· 508 508 509 509 w = nodes_weight(*maskp); 510 510 if (w) 511 - bit = bitmap_ord_to_pos(maskp->bits, 512 - get_random_int() % w, MAX_NUMNODES); 511 + bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w); 513 512 return bit; 514 513 #else 515 514 return 0;

+1 -4

kernel/sched/core.c

··· 357 357 /* 358 358 * Toggle the offline CPUs. 359 359 */ 360 - cpumask_copy(&sched_core_mask, cpu_possible_mask); 361 - cpumask_andnot(&sched_core_mask, &sched_core_mask, cpu_online_mask); 362 - 363 - for_each_cpu(cpu, &sched_core_mask) 360 + for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 364 361 cpu_rq(cpu)->core_enabled = enabled; 365 362 366 363 cpus_read_unlock();

+4 -2

kernel/smp.c

··· 1069 1069 int nr_cpus; 1070 1070 1071 1071 if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids) 1072 - nr_cpu_ids = nr_cpus; 1072 + set_nr_cpu_ids(nr_cpus); 1073 1073 1074 1074 return 0; 1075 1075 } ··· 1087 1087 1088 1088 early_param("maxcpus", maxcpus); 1089 1089 1090 + #if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS) 1090 1091 /* Setup number of possible processor ids */ 1091 1092 unsigned int nr_cpu_ids __read_mostly = NR_CPUS; 1092 1093 EXPORT_SYMBOL(nr_cpu_ids); 1094 + #endif 1093 1095 1094 1096 /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ 1095 1097 void __init setup_nr_cpu_ids(void) 1096 1098 { 1097 - nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; 1099 + set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); 1098 1100 } 1099 1101 1100 1102 /* Called by boot processor to activate the rest. */

+9

lib/Kconfig

··· 531 531 them on the stack. This is a bit more expensive, but avoids 532 532 stack overflow. 533 533 534 + config FORCE_NR_CPUS 535 + bool "NR_CPUS is set to an actual number of CPUs" 536 + depends on SMP 537 + help 538 + Say Yes if you have NR_CPUS set to an actual number of possible 539 + CPUs in your system, not to a default value. This forces the core 540 + code to rely on compile-time value and optimize kernel routines 541 + better. 542 + 534 543 config CPU_RMAP 535 544 bool 536 545 depends on SMP

+25 -43

lib/bitmap.c

··· 333 333 } 334 334 EXPORT_SYMBOL(__bitmap_subset); 335 335 336 + #define BITMAP_WEIGHT(FETCH, bits) \ 337 + ({ \ 338 + unsigned int __bits = (bits), idx, w = 0; \ 339 + \ 340 + for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \ 341 + w += hweight_long(FETCH); \ 342 + \ 343 + if (__bits % BITS_PER_LONG) \ 344 + w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \ 345 + \ 346 + w; \ 347 + }) 348 + 336 349 unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) 337 350 { 338 - unsigned int k, lim = bits/BITS_PER_LONG, w = 0; 339 - 340 - for (k = 0; k < lim; k++) 341 - w += hweight_long(bitmap[k]); 342 - 343 - if (bits % BITS_PER_LONG) 344 - w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits)); 345 - 346 - return w; 351 + return BITMAP_WEIGHT(bitmap[idx], bits); 347 352 } 348 353 EXPORT_SYMBOL(__bitmap_weight); 354 + 355 + unsigned int __bitmap_weight_and(const unsigned long *bitmap1, 356 + const unsigned long *bitmap2, unsigned int bits) 357 + { 358 + return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits); 359 + } 360 + EXPORT_SYMBOL(__bitmap_weight_and); 349 361 350 362 void __bitmap_set(unsigned long *map, unsigned int start, int len) 351 363 { ··· 965 953 if (pos >= nbits || !test_bit(pos, buf)) 966 954 return -1; 967 955 968 - return __bitmap_weight(buf, pos); 969 - } 970 - 971 - /** 972 - * bitmap_ord_to_pos - find position of n-th set bit in bitmap 973 - * @buf: pointer to bitmap 974 - * @ord: ordinal bit position (n-th set bit, n >= 0) 975 - * @nbits: number of valid bit positions in @buf 976 - * 977 - * Map the ordinal offset of bit @ord in @buf to its position in @buf. 978 - * Value of @ord should be in range 0 <= @ord < weight(buf). If @ord 979 - * >= weight(buf), returns @nbits. 980 - * 981 - * If for example, just bits 4 through 7 are set in @buf, then @ord 982 - * values 0 through 3 will get mapped to 4 through 7, respectively, 983 - * and all other @ord values returns @nbits. When @ord value 3 984 - * gets mapped to (returns) @pos value 7 in this example, that means 985 - * that the 3rd set bit (starting with 0th) is at position 7 in @buf. 986 - * 987 - * The bit positions 0 through @nbits-1 are valid positions in @buf. 988 - */ 989 - unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits) 990 - { 991 - unsigned int pos; 992 - 993 - for (pos = find_first_bit(buf, nbits); 994 - pos < nbits && ord; 995 - pos = find_next_bit(buf, nbits, pos + 1)) 996 - ord--; 997 - 998 - return pos; 956 + return bitmap_weight(buf, pos); 999 957 } 1000 958 1001 959 /** ··· 1017 1035 if (n < 0 || w == 0) 1018 1036 set_bit(oldbit, dst); /* identity map */ 1019 1037 else 1020 - set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); 1038 + set_bit(find_nth_bit(new, nbits, n % w), dst); 1021 1039 } 1022 1040 } 1023 1041 EXPORT_SYMBOL(bitmap_remap); ··· 1056 1074 if (n < 0 || w == 0) 1057 1075 return oldbit; 1058 1076 else 1059 - return bitmap_ord_to_pos(new, n % w, bits); 1077 + return find_nth_bit(new, bits, n % w); 1060 1078 } 1061 1079 EXPORT_SYMBOL(bitmap_bitremap); 1062 1080 ··· 1180 1198 * The following code is a more efficient, but less 1181 1199 * obvious, equivalent to the loop: 1182 1200 * for (m = 0; m < bitmap_weight(relmap, bits); m++) { 1183 - * n = bitmap_ord_to_pos(orig, m, bits); 1201 + * n = find_nth_bit(orig, bits, m); 1184 1202 * if (test_bit(m, orig)) 1185 1203 * set_bit(n, dst); 1186 1204 * }

+14 -22

lib/cpumask.c

··· 128 128 i %= num_online_cpus(); 129 129 130 130 if (node == NUMA_NO_NODE) { 131 - for_each_cpu(cpu, cpu_online_mask) 132 - if (i-- == 0) 133 - return cpu; 131 + cpu = cpumask_nth(i, cpu_online_mask); 132 + if (cpu < nr_cpu_ids) 133 + return cpu; 134 134 } else { 135 135 /* NUMA first. */ 136 - for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask) 137 - if (i-- == 0) 138 - return cpu; 136 + cpu = cpumask_nth_and(i, cpu_online_mask, cpumask_of_node(node)); 137 + if (cpu < nr_cpu_ids) 138 + return cpu; 139 139 140 - for_each_cpu(cpu, cpu_online_mask) { 141 - /* Skip NUMA nodes, done above. */ 142 - if (cpumask_test_cpu(cpu, cpumask_of_node(node))) 143 - continue; 140 + i -= cpumask_weight_and(cpu_online_mask, cpumask_of_node(node)); 144 141 145 - if (i-- == 0) 146 - return cpu; 147 - } 142 + /* Skip NUMA nodes, done above. */ 143 + cpu = cpumask_nth_andnot(i, cpu_online_mask, cpumask_of_node(node)); 144 + if (cpu < nr_cpu_ids) 145 + return cpu; 148 146 } 149 147 BUG(); 150 148 } ··· 166 168 /* NOTE: our first selection will skip 0. */ 167 169 prev = __this_cpu_read(distribute_cpu_mask_prev); 168 170 169 - next = cpumask_next_and(prev, src1p, src2p); 170 - if (next >= nr_cpu_ids) 171 - next = cpumask_first_and(src1p, src2p); 172 - 171 + next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), 172 + nr_cpumask_bits, prev + 1); 173 173 if (next < nr_cpu_ids) 174 174 __this_cpu_write(distribute_cpu_mask_prev, next); 175 175 ··· 181 185 182 186 /* NOTE: our first selection will skip 0. */ 183 187 prev = __this_cpu_read(distribute_cpu_mask_prev); 184 - 185 - next = cpumask_next(prev, srcp); 186 - if (next >= nr_cpu_ids) 187 - next = cpumask_first(srcp); 188 - 188 + next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1); 189 189 if (next < nr_cpu_ids) 190 190 __this_cpu_write(distribute_cpu_mask_prev, next); 191 191

+19

lib/cpumask_kunit.c

··· 33 33 KUNIT_EXPECT_EQ_MSG((test), nr_cpu_ids - mask_weight, iter, MASK_MSG(mask)); \ 34 34 } while (0) 35 35 36 + #define EXPECT_FOR_EACH_CPU_OP_EQ(test, op, mask1, mask2) \ 37 + do { \ 38 + const cpumask_t *m1 = (mask1); \ 39 + const cpumask_t *m2 = (mask2); \ 40 + int weight; \ 41 + int cpu, iter = 0; \ 42 + cpumask_##op(&mask_tmp, m1, m2); \ 43 + weight = cpumask_weight(&mask_tmp); \ 44 + for_each_cpu_##op(cpu, mask1, mask2) \ 45 + iter++; \ 46 + KUNIT_EXPECT_EQ((test), weight, iter); \ 47 + } while (0) 48 + 36 49 #define EXPECT_FOR_EACH_CPU_WRAP_EQ(test, mask) \ 37 50 do { \ 38 51 const cpumask_t *m = (mask); \ ··· 67 54 68 55 static cpumask_t mask_empty; 69 56 static cpumask_t mask_all; 57 + static cpumask_t mask_tmp; 70 58 71 59 static void test_cpumask_weight(struct kunit *test) 72 60 { ··· 115 101 EXPECT_FOR_EACH_CPU_EQ(test, &mask_empty); 116 102 EXPECT_FOR_EACH_CPU_NOT_EQ(test, &mask_empty); 117 103 EXPECT_FOR_EACH_CPU_WRAP_EQ(test, &mask_empty); 104 + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, &mask_empty, &mask_empty); 105 + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, &mask_empty); 106 + EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, &mask_empty, &mask_empty); 118 107 119 108 EXPECT_FOR_EACH_CPU_EQ(test, cpu_possible_mask); 120 109 EXPECT_FOR_EACH_CPU_NOT_EQ(test, cpu_possible_mask); 121 110 EXPECT_FOR_EACH_CPU_WRAP_EQ(test, cpu_possible_mask); 111 + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, cpu_possible_mask); 112 + EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, cpu_possible_mask, &mask_empty); 122 113 } 123 114 124 115 static void test_cpumask_iterators_builtin(struct kunit *test)

+161 -72

lib/find_bit.c

··· 19 19 #include <linux/minmax.h> 20 20 #include <linux/swab.h> 21 21 22 - #if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ 23 - !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ 24 - !defined(find_next_and_bit) 25 22 /* 26 - * This is a common helper function for find_next_bit, find_next_zero_bit, and 27 - * find_next_and_bit. The differences are: 28 - * - The "invert" argument, which is XORed with each fetched word before 29 - * searching it for one bits. 30 - * - The optional "addr2", which is anded with "addr1" if present. 23 + * Common helper for find_bit() function family 24 + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) 25 + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) 26 + * @size: The bitmap size in bits 31 27 */ 32 - unsigned long _find_next_bit(const unsigned long *addr1, 33 - const unsigned long *addr2, unsigned long nbits, 34 - unsigned long start, unsigned long invert, unsigned long le) 35 - { 36 - unsigned long tmp, mask; 28 + #define FIND_FIRST_BIT(FETCH, MUNGE, size) \ 29 + ({ \ 30 + unsigned long idx, val, sz = (size); \ 31 + \ 32 + for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ 33 + val = (FETCH); \ 34 + if (val) { \ 35 + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ 36 + break; \ 37 + } \ 38 + } \ 39 + \ 40 + sz; \ 41 + }) 37 42 38 - if (unlikely(start >= nbits)) 39 - return nbits; 43 + /* 44 + * Common helper for find_next_bit() function family 45 + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) 46 + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) 47 + * @size: The bitmap size in bits 48 + * @start: The bitnumber to start searching at 49 + */ 50 + #define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ 51 + ({ \ 52 + unsigned long mask, idx, tmp, sz = (size), __start = (start); \ 53 + \ 54 + if (unlikely(__start >= sz)) \ 55 + goto out; \ 56 + \ 57 + mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ 58 + idx = __start / BITS_PER_LONG; \ 59 + \ 60 + for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ 61 + if ((idx + 1) * BITS_PER_LONG >= sz) \ 62 + goto out; \ 63 + idx++; \ 64 + } \ 65 + \ 66 + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ 67 + out: \ 68 + sz; \ 69 + }) 40 70 41 - tmp = addr1[start / BITS_PER_LONG]; 42 - if (addr2) 43 - tmp &= addr2[start / BITS_PER_LONG]; 44 - tmp ^= invert; 45 - 46 - /* Handle 1st word. */ 47 - mask = BITMAP_FIRST_WORD_MASK(start); 48 - if (le) 49 - mask = swab(mask); 50 - 51 - tmp &= mask; 52 - 53 - start = round_down(start, BITS_PER_LONG); 54 - 55 - while (!tmp) { 56 - start += BITS_PER_LONG; 57 - if (start >= nbits) 58 - return nbits; 59 - 60 - tmp = addr1[start / BITS_PER_LONG]; 61 - if (addr2) 62 - tmp &= addr2[start / BITS_PER_LONG]; 63 - tmp ^= invert; 64 - } 65 - 66 - if (le) 67 - tmp = swab(tmp); 68 - 69 - return min(start + __ffs(tmp), nbits); 70 - } 71 - EXPORT_SYMBOL(_find_next_bit); 72 - #endif 71 + #define FIND_NTH_BIT(FETCH, size, num) \ 72 + ({ \ 73 + unsigned long sz = (size), nr = (num), idx, w, tmp; \ 74 + \ 75 + for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) { \ 76 + if (idx * BITS_PER_LONG + nr >= sz) \ 77 + goto out; \ 78 + \ 79 + tmp = (FETCH); \ 80 + w = hweight_long(tmp); \ 81 + if (w > nr) \ 82 + goto found; \ 83 + \ 84 + nr -= w; \ 85 + } \ 86 + \ 87 + if (sz % BITS_PER_LONG) \ 88 + tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz); \ 89 + found: \ 90 + sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz); \ 91 + out: \ 92 + sz; \ 93 + }) 73 94 74 95 #ifndef find_first_bit 75 96 /* ··· 98 77 */ 99 78 unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) 100 79 { 101 - unsigned long idx; 102 - 103 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 104 - if (addr[idx]) 105 - return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); 106 - } 107 - 108 - return size; 80 + return FIND_FIRST_BIT(addr[idx], /* nop */, size); 109 81 } 110 82 EXPORT_SYMBOL(_find_first_bit); 111 83 #endif ··· 111 97 const unsigned long *addr2, 112 98 unsigned long size) 113 99 { 114 - unsigned long idx, val; 115 - 116 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 117 - val = addr1[idx] & addr2[idx]; 118 - if (val) 119 - return min(idx * BITS_PER_LONG + __ffs(val), size); 120 - } 121 - 122 - return size; 100 + return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); 123 101 } 124 102 EXPORT_SYMBOL(_find_first_and_bit); 125 103 #endif ··· 122 116 */ 123 117 unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) 124 118 { 125 - unsigned long idx; 126 - 127 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 128 - if (addr[idx] != ~0UL) 129 - return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); 130 - } 131 - 132 - return size; 119 + return FIND_FIRST_BIT(~addr[idx], /* nop */, size); 133 120 } 134 121 EXPORT_SYMBOL(_find_first_zero_bit); 122 + #endif 123 + 124 + #ifndef find_next_bit 125 + unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) 126 + { 127 + return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); 128 + } 129 + EXPORT_SYMBOL(_find_next_bit); 130 + #endif 131 + 132 + unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) 133 + { 134 + return FIND_NTH_BIT(addr[idx], size, n); 135 + } 136 + EXPORT_SYMBOL(__find_nth_bit); 137 + 138 + unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, 139 + unsigned long size, unsigned long n) 140 + { 141 + return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n); 142 + } 143 + EXPORT_SYMBOL(__find_nth_and_bit); 144 + 145 + unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, 146 + unsigned long size, unsigned long n) 147 + { 148 + return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n); 149 + } 150 + EXPORT_SYMBOL(__find_nth_andnot_bit); 151 + 152 + #ifndef find_next_and_bit 153 + unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, 154 + unsigned long nbits, unsigned long start) 155 + { 156 + return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); 157 + } 158 + EXPORT_SYMBOL(_find_next_and_bit); 159 + #endif 160 + 161 + #ifndef find_next_andnot_bit 162 + unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, 163 + unsigned long nbits, unsigned long start) 164 + { 165 + return FIND_NEXT_BIT(addr1[idx] & ~addr2[idx], /* nop */, nbits, start); 166 + } 167 + EXPORT_SYMBOL(_find_next_andnot_bit); 168 + #endif 169 + 170 + #ifndef find_next_zero_bit 171 + unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, 172 + unsigned long start) 173 + { 174 + return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); 175 + } 176 + EXPORT_SYMBOL(_find_next_zero_bit); 135 177 #endif 136 178 137 179 #ifndef find_last_bit ··· 215 161 return offset; 216 162 } 217 163 EXPORT_SYMBOL(find_next_clump8); 164 + 165 + #ifdef __BIG_ENDIAN 166 + 167 + #ifndef find_first_zero_bit_le 168 + /* 169 + * Find the first cleared bit in an LE memory region. 170 + */ 171 + unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size) 172 + { 173 + return FIND_FIRST_BIT(~addr[idx], swab, size); 174 + } 175 + EXPORT_SYMBOL(_find_first_zero_bit_le); 176 + 177 + #endif 178 + 179 + #ifndef find_next_zero_bit_le 180 + unsigned long _find_next_zero_bit_le(const unsigned long *addr, 181 + unsigned long size, unsigned long offset) 182 + { 183 + return FIND_NEXT_BIT(~addr[idx], swab, size, offset); 184 + } 185 + EXPORT_SYMBOL(_find_next_zero_bit_le); 186 + #endif 187 + 188 + #ifndef find_next_bit_le 189 + unsigned long _find_next_bit_le(const unsigned long *addr, 190 + unsigned long size, unsigned long offset) 191 + { 192 + return FIND_NEXT_BIT(addr[idx], swab, size, offset); 193 + } 194 + EXPORT_SYMBOL(_find_next_bit_le); 195 + 196 + #endif 197 + 198 + #endif /* __BIG_ENDIAN */

+18

lib/find_bit_benchmark.c

··· 115 115 return 0; 116 116 } 117 117 118 + static int __init test_find_nth_bit(const unsigned long *bitmap, unsigned long len) 119 + { 120 + unsigned long l, n, w = bitmap_weight(bitmap, len); 121 + ktime_t time; 122 + 123 + time = ktime_get(); 124 + for (n = 0; n < w; n++) { 125 + l = find_nth_bit(bitmap, len, n); 126 + WARN_ON(l >= len); 127 + } 128 + time = ktime_get() - time; 129 + pr_err("find_nth_bit: %18llu ns, %6ld iterations\n", time, w); 130 + 131 + return 0; 132 + } 133 + 118 134 static int __init test_find_next_and_bit(const void *bitmap, 119 135 const void *bitmap2, unsigned long len) 120 136 { ··· 158 142 test_find_next_bit(bitmap, BITMAP_LEN); 159 143 test_find_next_zero_bit(bitmap, BITMAP_LEN); 160 144 test_find_last_bit(bitmap, BITMAP_LEN); 145 + test_find_nth_bit(bitmap, BITMAP_LEN / 10); 161 146 162 147 /* 163 148 * test_find_first_bit() may take some time, so ··· 181 164 test_find_next_bit(bitmap, BITMAP_LEN); 182 165 test_find_next_zero_bit(bitmap, BITMAP_LEN); 183 166 test_find_last_bit(bitmap, BITMAP_LEN); 167 + test_find_nth_bit(bitmap, BITMAP_LEN); 184 168 test_find_first_bit(bitmap, BITMAP_LEN); 185 169 test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN); 186 170 test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);

+288 -3

lib/test_bitmap.c

··· 16 16 17 17 #include "../tools/testing/selftests/kselftest_module.h" 18 18 19 + #define EXP1_IN_BITS (sizeof(exp1) * 8) 20 + 19 21 KSTM_MODULE_GLOBALS(); 20 22 21 23 static char pbl_buffer[PAGE_SIZE] __initdata; ··· 219 217 /* Zeroing entire area */ 220 218 bitmap_zero(bmap, 1024); 221 219 expect_eq_pbl("", bmap, 1024); 220 + } 221 + 222 + static void __init test_find_nth_bit(void) 223 + { 224 + unsigned long b, bit, cnt = 0; 225 + DECLARE_BITMAP(bmap, 64 * 3); 226 + 227 + bitmap_zero(bmap, 64 * 3); 228 + __set_bit(10, bmap); 229 + __set_bit(20, bmap); 230 + __set_bit(30, bmap); 231 + __set_bit(40, bmap); 232 + __set_bit(50, bmap); 233 + __set_bit(60, bmap); 234 + __set_bit(80, bmap); 235 + __set_bit(123, bmap); 236 + 237 + expect_eq_uint(10, find_nth_bit(bmap, 64 * 3, 0)); 238 + expect_eq_uint(20, find_nth_bit(bmap, 64 * 3, 1)); 239 + expect_eq_uint(30, find_nth_bit(bmap, 64 * 3, 2)); 240 + expect_eq_uint(40, find_nth_bit(bmap, 64 * 3, 3)); 241 + expect_eq_uint(50, find_nth_bit(bmap, 64 * 3, 4)); 242 + expect_eq_uint(60, find_nth_bit(bmap, 64 * 3, 5)); 243 + expect_eq_uint(80, find_nth_bit(bmap, 64 * 3, 6)); 244 + expect_eq_uint(123, find_nth_bit(bmap, 64 * 3, 7)); 245 + expect_eq_uint(64 * 3, find_nth_bit(bmap, 64 * 3, 8)); 246 + 247 + expect_eq_uint(10, find_nth_bit(bmap, 64 * 3 - 1, 0)); 248 + expect_eq_uint(20, find_nth_bit(bmap, 64 * 3 - 1, 1)); 249 + expect_eq_uint(30, find_nth_bit(bmap, 64 * 3 - 1, 2)); 250 + expect_eq_uint(40, find_nth_bit(bmap, 64 * 3 - 1, 3)); 251 + expect_eq_uint(50, find_nth_bit(bmap, 64 * 3 - 1, 4)); 252 + expect_eq_uint(60, find_nth_bit(bmap, 64 * 3 - 1, 5)); 253 + expect_eq_uint(80, find_nth_bit(bmap, 64 * 3 - 1, 6)); 254 + expect_eq_uint(123, find_nth_bit(bmap, 64 * 3 - 1, 7)); 255 + expect_eq_uint(64 * 3 - 1, find_nth_bit(bmap, 64 * 3 - 1, 8)); 256 + 257 + for_each_set_bit(bit, exp1, EXP1_IN_BITS) { 258 + b = find_nth_bit(exp1, EXP1_IN_BITS, cnt++); 259 + expect_eq_uint(b, bit); 260 + } 222 261 } 223 262 224 263 static void __init test_fill_set(void) ··· 600 557 } 601 558 } 602 559 603 - #define EXP1_IN_BITS (sizeof(exp1) * 8) 604 - 605 560 static void __init test_bitmap_arr32(void) 606 561 { 607 562 unsigned int nbits, next_bit; ··· 724 683 725 684 for_each_set_clump8(start, clump, bits, CLUMP_EXP_NUMBITS) 726 685 expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump); 686 + } 687 + 688 + static void __init test_for_each_set_bit_wrap(void) 689 + { 690 + DECLARE_BITMAP(orig, 500); 691 + DECLARE_BITMAP(copy, 500); 692 + unsigned int wr, bit; 693 + 694 + bitmap_zero(orig, 500); 695 + 696 + /* Set individual bits */ 697 + for (bit = 0; bit < 500; bit += 10) 698 + bitmap_set(orig, bit, 1); 699 + 700 + /* Set range of bits */ 701 + bitmap_set(orig, 100, 50); 702 + 703 + for (wr = 0; wr < 500; wr++) { 704 + bitmap_zero(copy, 500); 705 + 706 + for_each_set_bit_wrap(bit, orig, 500, wr) 707 + bitmap_set(copy, bit, 1); 708 + 709 + expect_eq_bitmap(orig, copy, 500); 710 + } 711 + } 712 + 713 + static void __init test_for_each_set_bit(void) 714 + { 715 + DECLARE_BITMAP(orig, 500); 716 + DECLARE_BITMAP(copy, 500); 717 + unsigned int bit; 718 + 719 + bitmap_zero(orig, 500); 720 + bitmap_zero(copy, 500); 721 + 722 + /* Set individual bits */ 723 + for (bit = 0; bit < 500; bit += 10) 724 + bitmap_set(orig, bit, 1); 725 + 726 + /* Set range of bits */ 727 + bitmap_set(orig, 100, 50); 728 + 729 + for_each_set_bit(bit, orig, 500) 730 + bitmap_set(copy, bit, 1); 731 + 732 + expect_eq_bitmap(orig, copy, 500); 733 + } 734 + 735 + static void __init test_for_each_set_bit_from(void) 736 + { 737 + DECLARE_BITMAP(orig, 500); 738 + DECLARE_BITMAP(copy, 500); 739 + unsigned int wr, bit; 740 + 741 + bitmap_zero(orig, 500); 742 + 743 + /* Set individual bits */ 744 + for (bit = 0; bit < 500; bit += 10) 745 + bitmap_set(orig, bit, 1); 746 + 747 + /* Set range of bits */ 748 + bitmap_set(orig, 100, 50); 749 + 750 + for (wr = 0; wr < 500; wr++) { 751 + DECLARE_BITMAP(tmp, 500); 752 + 753 + bitmap_zero(copy, 500); 754 + bit = wr; 755 + 756 + for_each_set_bit_from(bit, orig, 500) 757 + bitmap_set(copy, bit, 1); 758 + 759 + bitmap_copy(tmp, orig, 500); 760 + bitmap_clear(tmp, 0, wr); 761 + expect_eq_bitmap(tmp, copy, 500); 762 + } 763 + } 764 + 765 + static void __init test_for_each_clear_bit(void) 766 + { 767 + DECLARE_BITMAP(orig, 500); 768 + DECLARE_BITMAP(copy, 500); 769 + unsigned int bit; 770 + 771 + bitmap_fill(orig, 500); 772 + bitmap_fill(copy, 500); 773 + 774 + /* Set individual bits */ 775 + for (bit = 0; bit < 500; bit += 10) 776 + bitmap_clear(orig, bit, 1); 777 + 778 + /* Set range of bits */ 779 + bitmap_clear(orig, 100, 50); 780 + 781 + for_each_clear_bit(bit, orig, 500) 782 + bitmap_clear(copy, bit, 1); 783 + 784 + expect_eq_bitmap(orig, copy, 500); 785 + } 786 + 787 + static void __init test_for_each_clear_bit_from(void) 788 + { 789 + DECLARE_BITMAP(orig, 500); 790 + DECLARE_BITMAP(copy, 500); 791 + unsigned int wr, bit; 792 + 793 + bitmap_fill(orig, 500); 794 + 795 + /* Set individual bits */ 796 + for (bit = 0; bit < 500; bit += 10) 797 + bitmap_clear(orig, bit, 1); 798 + 799 + /* Set range of bits */ 800 + bitmap_clear(orig, 100, 50); 801 + 802 + for (wr = 0; wr < 500; wr++) { 803 + DECLARE_BITMAP(tmp, 500); 804 + 805 + bitmap_fill(copy, 500); 806 + bit = wr; 807 + 808 + for_each_clear_bit_from(bit, orig, 500) 809 + bitmap_clear(copy, bit, 1); 810 + 811 + bitmap_copy(tmp, orig, 500); 812 + bitmap_set(tmp, 0, wr); 813 + expect_eq_bitmap(tmp, copy, 500); 814 + } 815 + } 816 + 817 + static void __init test_for_each_set_bitrange(void) 818 + { 819 + DECLARE_BITMAP(orig, 500); 820 + DECLARE_BITMAP(copy, 500); 821 + unsigned int s, e; 822 + 823 + bitmap_zero(orig, 500); 824 + bitmap_zero(copy, 500); 825 + 826 + /* Set individual bits */ 827 + for (s = 0; s < 500; s += 10) 828 + bitmap_set(orig, s, 1); 829 + 830 + /* Set range of bits */ 831 + bitmap_set(orig, 100, 50); 832 + 833 + for_each_set_bitrange(s, e, orig, 500) 834 + bitmap_set(copy, s, e-s); 835 + 836 + expect_eq_bitmap(orig, copy, 500); 837 + } 838 + 839 + static void __init test_for_each_clear_bitrange(void) 840 + { 841 + DECLARE_BITMAP(orig, 500); 842 + DECLARE_BITMAP(copy, 500); 843 + unsigned int s, e; 844 + 845 + bitmap_fill(orig, 500); 846 + bitmap_fill(copy, 500); 847 + 848 + /* Set individual bits */ 849 + for (s = 0; s < 500; s += 10) 850 + bitmap_clear(orig, s, 1); 851 + 852 + /* Set range of bits */ 853 + bitmap_clear(orig, 100, 50); 854 + 855 + for_each_clear_bitrange(s, e, orig, 500) 856 + bitmap_clear(copy, s, e-s); 857 + 858 + expect_eq_bitmap(orig, copy, 500); 859 + } 860 + 861 + static void __init test_for_each_set_bitrange_from(void) 862 + { 863 + DECLARE_BITMAP(orig, 500); 864 + DECLARE_BITMAP(copy, 500); 865 + unsigned int wr, s, e; 866 + 867 + bitmap_zero(orig, 500); 868 + 869 + /* Set individual bits */ 870 + for (s = 0; s < 500; s += 10) 871 + bitmap_set(orig, s, 1); 872 + 873 + /* Set range of bits */ 874 + bitmap_set(orig, 100, 50); 875 + 876 + for (wr = 0; wr < 500; wr++) { 877 + DECLARE_BITMAP(tmp, 500); 878 + 879 + bitmap_zero(copy, 500); 880 + s = wr; 881 + 882 + for_each_set_bitrange_from(s, e, orig, 500) 883 + bitmap_set(copy, s, e - s); 884 + 885 + bitmap_copy(tmp, orig, 500); 886 + bitmap_clear(tmp, 0, wr); 887 + expect_eq_bitmap(tmp, copy, 500); 888 + } 889 + } 890 + 891 + static void __init test_for_each_clear_bitrange_from(void) 892 + { 893 + DECLARE_BITMAP(orig, 500); 894 + DECLARE_BITMAP(copy, 500); 895 + unsigned int wr, s, e; 896 + 897 + bitmap_fill(orig, 500); 898 + 899 + /* Set individual bits */ 900 + for (s = 0; s < 500; s += 10) 901 + bitmap_clear(orig, s, 1); 902 + 903 + /* Set range of bits */ 904 + bitmap_set(orig, 100, 50); 905 + 906 + for (wr = 0; wr < 500; wr++) { 907 + DECLARE_BITMAP(tmp, 500); 908 + 909 + bitmap_fill(copy, 500); 910 + s = wr; 911 + 912 + for_each_clear_bitrange_from(s, e, orig, 500) 913 + bitmap_clear(copy, s, e - s); 914 + 915 + bitmap_copy(tmp, orig, 500); 916 + bitmap_set(tmp, 0, wr); 917 + expect_eq_bitmap(tmp, copy, 500); 918 + } 727 919 } 728 920 729 921 struct test_bitmap_cut { ··· 1222 948 test_bitmap_parselist(); 1223 949 test_bitmap_printlist(); 1224 950 test_mem_optimisations(); 1225 - test_for_each_set_clump8(); 1226 951 test_bitmap_cut(); 1227 952 test_bitmap_print_buf(); 1228 953 test_bitmap_const_eval(); 954 + 955 + test_find_nth_bit(); 956 + test_for_each_set_bit(); 957 + test_for_each_set_bit_from(); 958 + test_for_each_clear_bit(); 959 + test_for_each_clear_bit_from(); 960 + test_for_each_set_bitrange(); 961 + test_for_each_clear_bitrange(); 962 + test_for_each_set_bitrange_from(); 963 + test_for_each_clear_bitrange_from(); 964 + test_for_each_set_clump8(); 965 + test_for_each_set_bit_wrap(); 1229 966 } 1230 967 1231 968 KSTM_MODULE_LOADERS(test_bitmap);

+12 -49

tools/include/linux/find.h

··· 8 8 9 9 #include <linux/bitops.h> 10 10 11 - extern unsigned long _find_next_bit(const unsigned long *addr1, 12 - const unsigned long *addr2, unsigned long nbits, 13 - unsigned long start, unsigned long invert, unsigned long le); 11 + unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, 12 + unsigned long start); 13 + unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, 14 + unsigned long nbits, unsigned long start); 15 + unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, 16 + unsigned long start); 14 17 extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); 15 18 extern unsigned long _find_first_and_bit(const unsigned long *addr1, 16 19 const unsigned long *addr2, unsigned long size); 17 20 extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); 18 - extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); 19 21 20 22 #ifndef find_next_bit 21 23 /** 22 24 * find_next_bit - find the next set bit in a memory region 23 25 * @addr: The address to base the search on 24 - * @offset: The bitnumber to start searching at 25 26 * @size: The bitmap size in bits 27 + * @offset: The bitnumber to start searching at 26 28 * 27 29 * Returns the bit number for the next set bit 28 30 * If no bits are set, returns @size. ··· 43 41 return val ? __ffs(val) : size; 44 42 } 45 43 46 - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); 44 + return _find_next_bit(addr, size, offset); 47 45 } 48 46 #endif 49 47 ··· 52 50 * find_next_and_bit - find the next set bit in both memory regions 53 51 * @addr1: The first address to base the search on 54 52 * @addr2: The second address to base the search on 55 - * @offset: The bitnumber to start searching at 56 53 * @size: The bitmap size in bits 54 + * @offset: The bitnumber to start searching at 57 55 * 58 56 * Returns the bit number for the next set bit 59 57 * If no bits are set, returns @size. ··· 73 71 return val ? __ffs(val) : size; 74 72 } 75 73 76 - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); 74 + return _find_next_and_bit(addr1, addr2, size, offset); 77 75 } 78 76 #endif 79 77 ··· 81 79 /** 82 80 * find_next_zero_bit - find the next cleared bit in a memory region 83 81 * @addr: The address to base the search on 84 - * @offset: The bitnumber to start searching at 85 82 * @size: The bitmap size in bits 83 + * @offset: The bitnumber to start searching at 86 84 * 87 85 * Returns the bit number of the next zero bit 88 86 * If no bits are zero, returns @size. ··· 101 99 return val == ~0UL ? size : ffz(val); 102 100 } 103 101 104 - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); 102 + return _find_next_zero_bit(addr, size, offset); 105 103 } 106 104 #endif 107 105 ··· 173 171 return _find_first_zero_bit(addr, size); 174 172 } 175 173 #endif 176 - 177 - #ifndef find_last_bit 178 - /** 179 - * find_last_bit - find the last set bit in a memory region 180 - * @addr: The address to start the search at 181 - * @size: The number of bits to search 182 - * 183 - * Returns the bit number of the last set bit, or size. 184 - */ 185 - static inline 186 - unsigned long find_last_bit(const unsigned long *addr, unsigned long size) 187 - { 188 - if (small_const_nbits(size)) { 189 - unsigned long val = *addr & GENMASK(size - 1, 0); 190 - 191 - return val ? __fls(val) : size; 192 - } 193 - 194 - return _find_last_bit(addr, size); 195 - } 196 - #endif 197 - 198 - /** 199 - * find_next_clump8 - find next 8-bit clump with set bits in a memory region 200 - * @clump: location to store copy of found clump 201 - * @addr: address to base the search on 202 - * @size: bitmap size in number of bits 203 - * @offset: bit offset at which to start searching 204 - * 205 - * Returns the bit offset for the next set clump; the found clump value is 206 - * copied to the location pointed by @clump. If no bits are set, returns @size. 207 - */ 208 - extern unsigned long find_next_clump8(unsigned long *clump, 209 - const unsigned long *addr, 210 - unsigned long size, unsigned long offset); 211 - 212 - #define find_first_clump8(clump, bits, size) \ 213 - find_next_clump8((clump), (bits), (size), 0) 214 - 215 174 216 175 #endif /*__LINUX_FIND_H_ */

+69 -80

tools/lib/find_bit.c

··· 18 18 #include <linux/bitmap.h> 19 19 #include <linux/kernel.h> 20 20 21 - #if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ 22 - !defined(find_next_and_bit) 21 + /* 22 + * Common helper for find_bit() function family 23 + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) 24 + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) 25 + * @size: The bitmap size in bits 26 + */ 27 + #define FIND_FIRST_BIT(FETCH, MUNGE, size) \ 28 + ({ \ 29 + unsigned long idx, val, sz = (size); \ 30 + \ 31 + for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ 32 + val = (FETCH); \ 33 + if (val) { \ 34 + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ 35 + break; \ 36 + } \ 37 + } \ 38 + \ 39 + sz; \ 40 + }) 23 41 24 42 /* 25 - * This is a common helper function for find_next_bit, find_next_zero_bit, and 26 - * find_next_and_bit. The differences are: 27 - * - The "invert" argument, which is XORed with each fetched word before 28 - * searching it for one bits. 29 - * - The optional "addr2", which is anded with "addr1" if present. 43 + * Common helper for find_next_bit() function family 44 + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) 45 + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) 46 + * @size: The bitmap size in bits 47 + * @start: The bitnumber to start searching at 30 48 */ 31 - unsigned long _find_next_bit(const unsigned long *addr1, 32 - const unsigned long *addr2, unsigned long nbits, 33 - unsigned long start, unsigned long invert, unsigned long le) 34 - { 35 - unsigned long tmp, mask; 36 - (void) le; 37 - 38 - if (unlikely(start >= nbits)) 39 - return nbits; 40 - 41 - tmp = addr1[start / BITS_PER_LONG]; 42 - if (addr2) 43 - tmp &= addr2[start / BITS_PER_LONG]; 44 - tmp ^= invert; 45 - 46 - /* Handle 1st word. */ 47 - mask = BITMAP_FIRST_WORD_MASK(start); 48 - 49 - /* 50 - * Due to the lack of swab() in tools, and the fact that it doesn't 51 - * need little-endian support, just comment it out 52 - */ 53 - #if (0) 54 - if (le) 55 - mask = swab(mask); 56 - #endif 57 - 58 - tmp &= mask; 59 - 60 - start = round_down(start, BITS_PER_LONG); 61 - 62 - while (!tmp) { 63 - start += BITS_PER_LONG; 64 - if (start >= nbits) 65 - return nbits; 66 - 67 - tmp = addr1[start / BITS_PER_LONG]; 68 - if (addr2) 69 - tmp &= addr2[start / BITS_PER_LONG]; 70 - tmp ^= invert; 71 - } 72 - 73 - #if (0) 74 - if (le) 75 - tmp = swab(tmp); 76 - #endif 77 - 78 - return min(start + __ffs(tmp), nbits); 79 - } 80 - #endif 49 + #define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ 50 + ({ \ 51 + unsigned long mask, idx, tmp, sz = (size), __start = (start); \ 52 + \ 53 + if (unlikely(__start >= sz)) \ 54 + goto out; \ 55 + \ 56 + mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ 57 + idx = __start / BITS_PER_LONG; \ 58 + \ 59 + for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ 60 + if ((idx + 1) * BITS_PER_LONG >= sz) \ 61 + goto out; \ 62 + idx++; \ 63 + } \ 64 + \ 65 + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ 66 + out: \ 67 + sz; \ 68 + }) 81 69 82 70 #ifndef find_first_bit 83 71 /* ··· 73 85 */ 74 86 unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) 75 87 { 76 - unsigned long idx; 77 - 78 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 79 - if (addr[idx]) 80 - return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); 81 - } 82 - 83 - return size; 88 + return FIND_FIRST_BIT(addr[idx], /* nop */, size); 84 89 } 85 90 #endif 86 91 ··· 85 104 const unsigned long *addr2, 86 105 unsigned long size) 87 106 { 88 - unsigned long idx, val; 89 - 90 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 91 - val = addr1[idx] & addr2[idx]; 92 - if (val) 93 - return min(idx * BITS_PER_LONG + __ffs(val), size); 94 - } 95 - 96 - return size; 107 + return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); 97 108 } 98 109 #endif 99 110 ··· 95 122 */ 96 123 unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) 97 124 { 98 - unsigned long idx; 125 + return FIND_FIRST_BIT(~addr[idx], /* nop */, size); 126 + } 127 + #endif 99 128 100 - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { 101 - if (addr[idx] != ~0UL) 102 - return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); 103 - } 129 + #ifndef find_next_bit 130 + unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) 131 + { 132 + return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); 133 + } 134 + #endif 104 135 105 - return size; 136 + #ifndef find_next_and_bit 137 + unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, 138 + unsigned long nbits, unsigned long start) 139 + { 140 + return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); 141 + } 142 + #endif 143 + 144 + #ifndef find_next_zero_bit 145 + unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, 146 + unsigned long start) 147 + { 148 + return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); 106 149 } 107 150 #endif

Configure Feed

Configure Feed