+5 -1

MAINTAINERS

reviewed

··· 3988 3988 ATOMIC INFRASTRUCTURE 3989 3989 M: Will Deacon <will@kernel.org> 3990 3990 M: Peter Zijlstra <peterz@infradead.org> 3991 3991 - R: Boqun Feng <boqun.feng@gmail.com> 3991 3991 + M: Boqun Feng <boqun.feng@gmail.com> 3992 3992 R: Mark Rutland <mark.rutland@arm.com> 3993 3993 + R: Gary Guo <gary@garyguo.net> 3993 3994 L: linux-kernel@vger.kernel.org 3994 3995 S: Maintained 3995 3996 F: Documentation/atomic_*.txt ··· 3998 3997 F: include/*/atomic*.h 3999 3998 F: include/linux/refcount.h 4000 3999 F: scripts/atomic/ 4000 4000 + F: rust/kernel/sync/atomic.rs 4001 4001 + F: rust/kernel/sync/atomic/ 4002 4002 + F: rust/kernel/sync/refcount.rs 4001 4003 4002 4004 ATTO EXPRESSSAS SAS/SATA RAID SCSI DRIVER 4003 4005 M: Bradley Grove <linuxdrivers@attotech.com>

+1040

rust/helpers/atomic.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + // Generated by scripts/atomic/gen-rust-atomic-helpers.sh 4 4 + // DO NOT MODIFY THIS FILE DIRECTLY 5 5 + 6 6 + /* 7 7 + * This file provides helpers for the various atomic functions for Rust. 8 8 + */ 9 9 + #ifndef _RUST_ATOMIC_API_H 10 10 + #define _RUST_ATOMIC_API_H 11 11 + 12 12 + #include <linux/atomic.h> 13 13 + 14 14 + // TODO: Remove this after INLINE_HELPERS support is added. 15 15 + #ifndef __rust_helper 16 16 + #define __rust_helper 17 17 + #endif 18 18 + 19 19 + __rust_helper int 20 20 + rust_helper_atomic_read(const atomic_t *v) 21 21 + { 22 22 + return atomic_read(v); 23 23 + } 24 24 + 25 25 + __rust_helper int 26 26 + rust_helper_atomic_read_acquire(const atomic_t *v) 27 27 + { 28 28 + return atomic_read_acquire(v); 29 29 + } 30 30 + 31 31 + __rust_helper void 32 32 + rust_helper_atomic_set(atomic_t *v, int i) 33 33 + { 34 34 + atomic_set(v, i); 35 35 + } 36 36 + 37 37 + __rust_helper void 38 38 + rust_helper_atomic_set_release(atomic_t *v, int i) 39 39 + { 40 40 + atomic_set_release(v, i); 41 41 + } 42 42 + 43 43 + __rust_helper void 44 44 + rust_helper_atomic_add(int i, atomic_t *v) 45 45 + { 46 46 + atomic_add(i, v); 47 47 + } 48 48 + 49 49 + __rust_helper int 50 50 + rust_helper_atomic_add_return(int i, atomic_t *v) 51 51 + { 52 52 + return atomic_add_return(i, v); 53 53 + } 54 54 + 55 55 + __rust_helper int 56 56 + rust_helper_atomic_add_return_acquire(int i, atomic_t *v) 57 57 + { 58 58 + return atomic_add_return_acquire(i, v); 59 59 + } 60 60 + 61 61 + __rust_helper int 62 62 + rust_helper_atomic_add_return_release(int i, atomic_t *v) 63 63 + { 64 64 + return atomic_add_return_release(i, v); 65 65 + } 66 66 + 67 67 + __rust_helper int 68 68 + rust_helper_atomic_add_return_relaxed(int i, atomic_t *v) 69 69 + { 70 70 + return atomic_add_return_relaxed(i, v); 71 71 + } 72 72 + 73 73 + __rust_helper int 74 74 + rust_helper_atomic_fetch_add(int i, atomic_t *v) 75 75 + { 76 76 + return atomic_fetch_add(i, v); 77 77 + } 78 78 + 79 79 + __rust_helper int 80 80 + rust_helper_atomic_fetch_add_acquire(int i, atomic_t *v) 81 81 + { 82 82 + return atomic_fetch_add_acquire(i, v); 83 83 + } 84 84 + 85 85 + __rust_helper int 86 86 + rust_helper_atomic_fetch_add_release(int i, atomic_t *v) 87 87 + { 88 88 + return atomic_fetch_add_release(i, v); 89 89 + } 90 90 + 91 91 + __rust_helper int 92 92 + rust_helper_atomic_fetch_add_relaxed(int i, atomic_t *v) 93 93 + { 94 94 + return atomic_fetch_add_relaxed(i, v); 95 95 + } 96 96 + 97 97 + __rust_helper void 98 98 + rust_helper_atomic_sub(int i, atomic_t *v) 99 99 + { 100 100 + atomic_sub(i, v); 101 101 + } 102 102 + 103 103 + __rust_helper int 104 104 + rust_helper_atomic_sub_return(int i, atomic_t *v) 105 105 + { 106 106 + return atomic_sub_return(i, v); 107 107 + } 108 108 + 109 109 + __rust_helper int 110 110 + rust_helper_atomic_sub_return_acquire(int i, atomic_t *v) 111 111 + { 112 112 + return atomic_sub_return_acquire(i, v); 113 113 + } 114 114 + 115 115 + __rust_helper int 116 116 + rust_helper_atomic_sub_return_release(int i, atomic_t *v) 117 117 + { 118 118 + return atomic_sub_return_release(i, v); 119 119 + } 120 120 + 121 121 + __rust_helper int 122 122 + rust_helper_atomic_sub_return_relaxed(int i, atomic_t *v) 123 123 + { 124 124 + return atomic_sub_return_relaxed(i, v); 125 125 + } 126 126 + 127 127 + __rust_helper int 128 128 + rust_helper_atomic_fetch_sub(int i, atomic_t *v) 129 129 + { 130 130 + return atomic_fetch_sub(i, v); 131 131 + } 132 132 + 133 133 + __rust_helper int 134 134 + rust_helper_atomic_fetch_sub_acquire(int i, atomic_t *v) 135 135 + { 136 136 + return atomic_fetch_sub_acquire(i, v); 137 137 + } 138 138 + 139 139 + __rust_helper int 140 140 + rust_helper_atomic_fetch_sub_release(int i, atomic_t *v) 141 141 + { 142 142 + return atomic_fetch_sub_release(i, v); 143 143 + } 144 144 + 145 145 + __rust_helper int 146 146 + rust_helper_atomic_fetch_sub_relaxed(int i, atomic_t *v) 147 147 + { 148 148 + return atomic_fetch_sub_relaxed(i, v); 149 149 + } 150 150 + 151 151 + __rust_helper void 152 152 + rust_helper_atomic_inc(atomic_t *v) 153 153 + { 154 154 + atomic_inc(v); 155 155 + } 156 156 + 157 157 + __rust_helper int 158 158 + rust_helper_atomic_inc_return(atomic_t *v) 159 159 + { 160 160 + return atomic_inc_return(v); 161 161 + } 162 162 + 163 163 + __rust_helper int 164 164 + rust_helper_atomic_inc_return_acquire(atomic_t *v) 165 165 + { 166 166 + return atomic_inc_return_acquire(v); 167 167 + } 168 168 + 169 169 + __rust_helper int 170 170 + rust_helper_atomic_inc_return_release(atomic_t *v) 171 171 + { 172 172 + return atomic_inc_return_release(v); 173 173 + } 174 174 + 175 175 + __rust_helper int 176 176 + rust_helper_atomic_inc_return_relaxed(atomic_t *v) 177 177 + { 178 178 + return atomic_inc_return_relaxed(v); 179 179 + } 180 180 + 181 181 + __rust_helper int 182 182 + rust_helper_atomic_fetch_inc(atomic_t *v) 183 183 + { 184 184 + return atomic_fetch_inc(v); 185 185 + } 186 186 + 187 187 + __rust_helper int 188 188 + rust_helper_atomic_fetch_inc_acquire(atomic_t *v) 189 189 + { 190 190 + return atomic_fetch_inc_acquire(v); 191 191 + } 192 192 + 193 193 + __rust_helper int 194 194 + rust_helper_atomic_fetch_inc_release(atomic_t *v) 195 195 + { 196 196 + return atomic_fetch_inc_release(v); 197 197 + } 198 198 + 199 199 + __rust_helper int 200 200 + rust_helper_atomic_fetch_inc_relaxed(atomic_t *v) 201 201 + { 202 202 + return atomic_fetch_inc_relaxed(v); 203 203 + } 204 204 + 205 205 + __rust_helper void 206 206 + rust_helper_atomic_dec(atomic_t *v) 207 207 + { 208 208 + atomic_dec(v); 209 209 + } 210 210 + 211 211 + __rust_helper int 212 212 + rust_helper_atomic_dec_return(atomic_t *v) 213 213 + { 214 214 + return atomic_dec_return(v); 215 215 + } 216 216 + 217 217 + __rust_helper int 218 218 + rust_helper_atomic_dec_return_acquire(atomic_t *v) 219 219 + { 220 220 + return atomic_dec_return_acquire(v); 221 221 + } 222 222 + 223 223 + __rust_helper int 224 224 + rust_helper_atomic_dec_return_release(atomic_t *v) 225 225 + { 226 226 + return atomic_dec_return_release(v); 227 227 + } 228 228 + 229 229 + __rust_helper int 230 230 + rust_helper_atomic_dec_return_relaxed(atomic_t *v) 231 231 + { 232 232 + return atomic_dec_return_relaxed(v); 233 233 + } 234 234 + 235 235 + __rust_helper int 236 236 + rust_helper_atomic_fetch_dec(atomic_t *v) 237 237 + { 238 238 + return atomic_fetch_dec(v); 239 239 + } 240 240 + 241 241 + __rust_helper int 242 242 + rust_helper_atomic_fetch_dec_acquire(atomic_t *v) 243 243 + { 244 244 + return atomic_fetch_dec_acquire(v); 245 245 + } 246 246 + 247 247 + __rust_helper int 248 248 + rust_helper_atomic_fetch_dec_release(atomic_t *v) 249 249 + { 250 250 + return atomic_fetch_dec_release(v); 251 251 + } 252 252 + 253 253 + __rust_helper int 254 254 + rust_helper_atomic_fetch_dec_relaxed(atomic_t *v) 255 255 + { 256 256 + return atomic_fetch_dec_relaxed(v); 257 257 + } 258 258 + 259 259 + __rust_helper void 260 260 + rust_helper_atomic_and(int i, atomic_t *v) 261 261 + { 262 262 + atomic_and(i, v); 263 263 + } 264 264 + 265 265 + __rust_helper int 266 266 + rust_helper_atomic_fetch_and(int i, atomic_t *v) 267 267 + { 268 268 + return atomic_fetch_and(i, v); 269 269 + } 270 270 + 271 271 + __rust_helper int 272 272 + rust_helper_atomic_fetch_and_acquire(int i, atomic_t *v) 273 273 + { 274 274 + return atomic_fetch_and_acquire(i, v); 275 275 + } 276 276 + 277 277 + __rust_helper int 278 278 + rust_helper_atomic_fetch_and_release(int i, atomic_t *v) 279 279 + { 280 280 + return atomic_fetch_and_release(i, v); 281 281 + } 282 282 + 283 283 + __rust_helper int 284 284 + rust_helper_atomic_fetch_and_relaxed(int i, atomic_t *v) 285 285 + { 286 286 + return atomic_fetch_and_relaxed(i, v); 287 287 + } 288 288 + 289 289 + __rust_helper void 290 290 + rust_helper_atomic_andnot(int i, atomic_t *v) 291 291 + { 292 292 + atomic_andnot(i, v); 293 293 + } 294 294 + 295 295 + __rust_helper int 296 296 + rust_helper_atomic_fetch_andnot(int i, atomic_t *v) 297 297 + { 298 298 + return atomic_fetch_andnot(i, v); 299 299 + } 300 300 + 301 301 + __rust_helper int 302 302 + rust_helper_atomic_fetch_andnot_acquire(int i, atomic_t *v) 303 303 + { 304 304 + return atomic_fetch_andnot_acquire(i, v); 305 305 + } 306 306 + 307 307 + __rust_helper int 308 308 + rust_helper_atomic_fetch_andnot_release(int i, atomic_t *v) 309 309 + { 310 310 + return atomic_fetch_andnot_release(i, v); 311 311 + } 312 312 + 313 313 + __rust_helper int 314 314 + rust_helper_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 315 315 + { 316 316 + return atomic_fetch_andnot_relaxed(i, v); 317 317 + } 318 318 + 319 319 + __rust_helper void 320 320 + rust_helper_atomic_or(int i, atomic_t *v) 321 321 + { 322 322 + atomic_or(i, v); 323 323 + } 324 324 + 325 325 + __rust_helper int 326 326 + rust_helper_atomic_fetch_or(int i, atomic_t *v) 327 327 + { 328 328 + return atomic_fetch_or(i, v); 329 329 + } 330 330 + 331 331 + __rust_helper int 332 332 + rust_helper_atomic_fetch_or_acquire(int i, atomic_t *v) 333 333 + { 334 334 + return atomic_fetch_or_acquire(i, v); 335 335 + } 336 336 + 337 337 + __rust_helper int 338 338 + rust_helper_atomic_fetch_or_release(int i, atomic_t *v) 339 339 + { 340 340 + return atomic_fetch_or_release(i, v); 341 341 + } 342 342 + 343 343 + __rust_helper int 344 344 + rust_helper_atomic_fetch_or_relaxed(int i, atomic_t *v) 345 345 + { 346 346 + return atomic_fetch_or_relaxed(i, v); 347 347 + } 348 348 + 349 349 + __rust_helper void 350 350 + rust_helper_atomic_xor(int i, atomic_t *v) 351 351 + { 352 352 + atomic_xor(i, v); 353 353 + } 354 354 + 355 355 + __rust_helper int 356 356 + rust_helper_atomic_fetch_xor(int i, atomic_t *v) 357 357 + { 358 358 + return atomic_fetch_xor(i, v); 359 359 + } 360 360 + 361 361 + __rust_helper int 362 362 + rust_helper_atomic_fetch_xor_acquire(int i, atomic_t *v) 363 363 + { 364 364 + return atomic_fetch_xor_acquire(i, v); 365 365 + } 366 366 + 367 367 + __rust_helper int 368 368 + rust_helper_atomic_fetch_xor_release(int i, atomic_t *v) 369 369 + { 370 370 + return atomic_fetch_xor_release(i, v); 371 371 + } 372 372 + 373 373 + __rust_helper int 374 374 + rust_helper_atomic_fetch_xor_relaxed(int i, atomic_t *v) 375 375 + { 376 376 + return atomic_fetch_xor_relaxed(i, v); 377 377 + } 378 378 + 379 379 + __rust_helper int 380 380 + rust_helper_atomic_xchg(atomic_t *v, int new) 381 381 + { 382 382 + return atomic_xchg(v, new); 383 383 + } 384 384 + 385 385 + __rust_helper int 386 386 + rust_helper_atomic_xchg_acquire(atomic_t *v, int new) 387 387 + { 388 388 + return atomic_xchg_acquire(v, new); 389 389 + } 390 390 + 391 391 + __rust_helper int 392 392 + rust_helper_atomic_xchg_release(atomic_t *v, int new) 393 393 + { 394 394 + return atomic_xchg_release(v, new); 395 395 + } 396 396 + 397 397 + __rust_helper int 398 398 + rust_helper_atomic_xchg_relaxed(atomic_t *v, int new) 399 399 + { 400 400 + return atomic_xchg_relaxed(v, new); 401 401 + } 402 402 + 403 403 + __rust_helper int 404 404 + rust_helper_atomic_cmpxchg(atomic_t *v, int old, int new) 405 405 + { 406 406 + return atomic_cmpxchg(v, old, new); 407 407 + } 408 408 + 409 409 + __rust_helper int 410 410 + rust_helper_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 411 411 + { 412 412 + return atomic_cmpxchg_acquire(v, old, new); 413 413 + } 414 414 + 415 415 + __rust_helper int 416 416 + rust_helper_atomic_cmpxchg_release(atomic_t *v, int old, int new) 417 417 + { 418 418 + return atomic_cmpxchg_release(v, old, new); 419 419 + } 420 420 + 421 421 + __rust_helper int 422 422 + rust_helper_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 423 423 + { 424 424 + return atomic_cmpxchg_relaxed(v, old, new); 425 425 + } 426 426 + 427 427 + __rust_helper bool 428 428 + rust_helper_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 429 429 + { 430 430 + return atomic_try_cmpxchg(v, old, new); 431 431 + } 432 432 + 433 433 + __rust_helper bool 434 434 + rust_helper_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 435 435 + { 436 436 + return atomic_try_cmpxchg_acquire(v, old, new); 437 437 + } 438 438 + 439 439 + __rust_helper bool 440 440 + rust_helper_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 441 441 + { 442 442 + return atomic_try_cmpxchg_release(v, old, new); 443 443 + } 444 444 + 445 445 + __rust_helper bool 446 446 + rust_helper_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 447 447 + { 448 448 + return atomic_try_cmpxchg_relaxed(v, old, new); 449 449 + } 450 450 + 451 451 + __rust_helper bool 452 452 + rust_helper_atomic_sub_and_test(int i, atomic_t *v) 453 453 + { 454 454 + return atomic_sub_and_test(i, v); 455 455 + } 456 456 + 457 457 + __rust_helper bool 458 458 + rust_helper_atomic_dec_and_test(atomic_t *v) 459 459 + { 460 460 + return atomic_dec_and_test(v); 461 461 + } 462 462 + 463 463 + __rust_helper bool 464 464 + rust_helper_atomic_inc_and_test(atomic_t *v) 465 465 + { 466 466 + return atomic_inc_and_test(v); 467 467 + } 468 468 + 469 469 + __rust_helper bool 470 470 + rust_helper_atomic_add_negative(int i, atomic_t *v) 471 471 + { 472 472 + return atomic_add_negative(i, v); 473 473 + } 474 474 + 475 475 + __rust_helper bool 476 476 + rust_helper_atomic_add_negative_acquire(int i, atomic_t *v) 477 477 + { 478 478 + return atomic_add_negative_acquire(i, v); 479 479 + } 480 480 + 481 481 + __rust_helper bool 482 482 + rust_helper_atomic_add_negative_release(int i, atomic_t *v) 483 483 + { 484 484 + return atomic_add_negative_release(i, v); 485 485 + } 486 486 + 487 487 + __rust_helper bool 488 488 + rust_helper_atomic_add_negative_relaxed(int i, atomic_t *v) 489 489 + { 490 490 + return atomic_add_negative_relaxed(i, v); 491 491 + } 492 492 + 493 493 + __rust_helper int 494 494 + rust_helper_atomic_fetch_add_unless(atomic_t *v, int a, int u) 495 495 + { 496 496 + return atomic_fetch_add_unless(v, a, u); 497 497 + } 498 498 + 499 499 + __rust_helper bool 500 500 + rust_helper_atomic_add_unless(atomic_t *v, int a, int u) 501 501 + { 502 502 + return atomic_add_unless(v, a, u); 503 503 + } 504 504 + 505 505 + __rust_helper bool 506 506 + rust_helper_atomic_inc_not_zero(atomic_t *v) 507 507 + { 508 508 + return atomic_inc_not_zero(v); 509 509 + } 510 510 + 511 511 + __rust_helper bool 512 512 + rust_helper_atomic_inc_unless_negative(atomic_t *v) 513 513 + { 514 514 + return atomic_inc_unless_negative(v); 515 515 + } 516 516 + 517 517 + __rust_helper bool 518 518 + rust_helper_atomic_dec_unless_positive(atomic_t *v) 519 519 + { 520 520 + return atomic_dec_unless_positive(v); 521 521 + } 522 522 + 523 523 + __rust_helper int 524 524 + rust_helper_atomic_dec_if_positive(atomic_t *v) 525 525 + { 526 526 + return atomic_dec_if_positive(v); 527 527 + } 528 528 + 529 529 + __rust_helper s64 530 530 + rust_helper_atomic64_read(const atomic64_t *v) 531 531 + { 532 532 + return atomic64_read(v); 533 533 + } 534 534 + 535 535 + __rust_helper s64 536 536 + rust_helper_atomic64_read_acquire(const atomic64_t *v) 537 537 + { 538 538 + return atomic64_read_acquire(v); 539 539 + } 540 540 + 541 541 + __rust_helper void 542 542 + rust_helper_atomic64_set(atomic64_t *v, s64 i) 543 543 + { 544 544 + atomic64_set(v, i); 545 545 + } 546 546 + 547 547 + __rust_helper void 548 548 + rust_helper_atomic64_set_release(atomic64_t *v, s64 i) 549 549 + { 550 550 + atomic64_set_release(v, i); 551 551 + } 552 552 + 553 553 + __rust_helper void 554 554 + rust_helper_atomic64_add(s64 i, atomic64_t *v) 555 555 + { 556 556 + atomic64_add(i, v); 557 557 + } 558 558 + 559 559 + __rust_helper s64 560 560 + rust_helper_atomic64_add_return(s64 i, atomic64_t *v) 561 561 + { 562 562 + return atomic64_add_return(i, v); 563 563 + } 564 564 + 565 565 + __rust_helper s64 566 566 + rust_helper_atomic64_add_return_acquire(s64 i, atomic64_t *v) 567 567 + { 568 568 + return atomic64_add_return_acquire(i, v); 569 569 + } 570 570 + 571 571 + __rust_helper s64 572 572 + rust_helper_atomic64_add_return_release(s64 i, atomic64_t *v) 573 573 + { 574 574 + return atomic64_add_return_release(i, v); 575 575 + } 576 576 + 577 577 + __rust_helper s64 578 578 + rust_helper_atomic64_add_return_relaxed(s64 i, atomic64_t *v) 579 579 + { 580 580 + return atomic64_add_return_relaxed(i, v); 581 581 + } 582 582 + 583 583 + __rust_helper s64 584 584 + rust_helper_atomic64_fetch_add(s64 i, atomic64_t *v) 585 585 + { 586 586 + return atomic64_fetch_add(i, v); 587 587 + } 588 588 + 589 589 + __rust_helper s64 590 590 + rust_helper_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 591 591 + { 592 592 + return atomic64_fetch_add_acquire(i, v); 593 593 + } 594 594 + 595 595 + __rust_helper s64 596 596 + rust_helper_atomic64_fetch_add_release(s64 i, atomic64_t *v) 597 597 + { 598 598 + return atomic64_fetch_add_release(i, v); 599 599 + } 600 600 + 601 601 + __rust_helper s64 602 602 + rust_helper_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 603 603 + { 604 604 + return atomic64_fetch_add_relaxed(i, v); 605 605 + } 606 606 + 607 607 + __rust_helper void 608 608 + rust_helper_atomic64_sub(s64 i, atomic64_t *v) 609 609 + { 610 610 + atomic64_sub(i, v); 611 611 + } 612 612 + 613 613 + __rust_helper s64 614 614 + rust_helper_atomic64_sub_return(s64 i, atomic64_t *v) 615 615 + { 616 616 + return atomic64_sub_return(i, v); 617 617 + } 618 618 + 619 619 + __rust_helper s64 620 620 + rust_helper_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 621 621 + { 622 622 + return atomic64_sub_return_acquire(i, v); 623 623 + } 624 624 + 625 625 + __rust_helper s64 626 626 + rust_helper_atomic64_sub_return_release(s64 i, atomic64_t *v) 627 627 + { 628 628 + return atomic64_sub_return_release(i, v); 629 629 + } 630 630 + 631 631 + __rust_helper s64 632 632 + rust_helper_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 633 633 + { 634 634 + return atomic64_sub_return_relaxed(i, v); 635 635 + } 636 636 + 637 637 + __rust_helper s64 638 638 + rust_helper_atomic64_fetch_sub(s64 i, atomic64_t *v) 639 639 + { 640 640 + return atomic64_fetch_sub(i, v); 641 641 + } 642 642 + 643 643 + __rust_helper s64 644 644 + rust_helper_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 645 645 + { 646 646 + return atomic64_fetch_sub_acquire(i, v); 647 647 + } 648 648 + 649 649 + __rust_helper s64 650 650 + rust_helper_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 651 651 + { 652 652 + return atomic64_fetch_sub_release(i, v); 653 653 + } 654 654 + 655 655 + __rust_helper s64 656 656 + rust_helper_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 657 657 + { 658 658 + return atomic64_fetch_sub_relaxed(i, v); 659 659 + } 660 660 + 661 661 + __rust_helper void 662 662 + rust_helper_atomic64_inc(atomic64_t *v) 663 663 + { 664 664 + atomic64_inc(v); 665 665 + } 666 666 + 667 667 + __rust_helper s64 668 668 + rust_helper_atomic64_inc_return(atomic64_t *v) 669 669 + { 670 670 + return atomic64_inc_return(v); 671 671 + } 672 672 + 673 673 + __rust_helper s64 674 674 + rust_helper_atomic64_inc_return_acquire(atomic64_t *v) 675 675 + { 676 676 + return atomic64_inc_return_acquire(v); 677 677 + } 678 678 + 679 679 + __rust_helper s64 680 680 + rust_helper_atomic64_inc_return_release(atomic64_t *v) 681 681 + { 682 682 + return atomic64_inc_return_release(v); 683 683 + } 684 684 + 685 685 + __rust_helper s64 686 686 + rust_helper_atomic64_inc_return_relaxed(atomic64_t *v) 687 687 + { 688 688 + return atomic64_inc_return_relaxed(v); 689 689 + } 690 690 + 691 691 + __rust_helper s64 692 692 + rust_helper_atomic64_fetch_inc(atomic64_t *v) 693 693 + { 694 694 + return atomic64_fetch_inc(v); 695 695 + } 696 696 + 697 697 + __rust_helper s64 698 698 + rust_helper_atomic64_fetch_inc_acquire(atomic64_t *v) 699 699 + { 700 700 + return atomic64_fetch_inc_acquire(v); 701 701 + } 702 702 + 703 703 + __rust_helper s64 704 704 + rust_helper_atomic64_fetch_inc_release(atomic64_t *v) 705 705 + { 706 706 + return atomic64_fetch_inc_release(v); 707 707 + } 708 708 + 709 709 + __rust_helper s64 710 710 + rust_helper_atomic64_fetch_inc_relaxed(atomic64_t *v) 711 711 + { 712 712 + return atomic64_fetch_inc_relaxed(v); 713 713 + } 714 714 + 715 715 + __rust_helper void 716 716 + rust_helper_atomic64_dec(atomic64_t *v) 717 717 + { 718 718 + atomic64_dec(v); 719 719 + } 720 720 + 721 721 + __rust_helper s64 722 722 + rust_helper_atomic64_dec_return(atomic64_t *v) 723 723 + { 724 724 + return atomic64_dec_return(v); 725 725 + } 726 726 + 727 727 + __rust_helper s64 728 728 + rust_helper_atomic64_dec_return_acquire(atomic64_t *v) 729 729 + { 730 730 + return atomic64_dec_return_acquire(v); 731 731 + } 732 732 + 733 733 + __rust_helper s64 734 734 + rust_helper_atomic64_dec_return_release(atomic64_t *v) 735 735 + { 736 736 + return atomic64_dec_return_release(v); 737 737 + } 738 738 + 739 739 + __rust_helper s64 740 740 + rust_helper_atomic64_dec_return_relaxed(atomic64_t *v) 741 741 + { 742 742 + return atomic64_dec_return_relaxed(v); 743 743 + } 744 744 + 745 745 + __rust_helper s64 746 746 + rust_helper_atomic64_fetch_dec(atomic64_t *v) 747 747 + { 748 748 + return atomic64_fetch_dec(v); 749 749 + } 750 750 + 751 751 + __rust_helper s64 752 752 + rust_helper_atomic64_fetch_dec_acquire(atomic64_t *v) 753 753 + { 754 754 + return atomic64_fetch_dec_acquire(v); 755 755 + } 756 756 + 757 757 + __rust_helper s64 758 758 + rust_helper_atomic64_fetch_dec_release(atomic64_t *v) 759 759 + { 760 760 + return atomic64_fetch_dec_release(v); 761 761 + } 762 762 + 763 763 + __rust_helper s64 764 764 + rust_helper_atomic64_fetch_dec_relaxed(atomic64_t *v) 765 765 + { 766 766 + return atomic64_fetch_dec_relaxed(v); 767 767 + } 768 768 + 769 769 + __rust_helper void 770 770 + rust_helper_atomic64_and(s64 i, atomic64_t *v) 771 771 + { 772 772 + atomic64_and(i, v); 773 773 + } 774 774 + 775 775 + __rust_helper s64 776 776 + rust_helper_atomic64_fetch_and(s64 i, atomic64_t *v) 777 777 + { 778 778 + return atomic64_fetch_and(i, v); 779 779 + } 780 780 + 781 781 + __rust_helper s64 782 782 + rust_helper_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 783 783 + { 784 784 + return atomic64_fetch_and_acquire(i, v); 785 785 + } 786 786 + 787 787 + __rust_helper s64 788 788 + rust_helper_atomic64_fetch_and_release(s64 i, atomic64_t *v) 789 789 + { 790 790 + return atomic64_fetch_and_release(i, v); 791 791 + } 792 792 + 793 793 + __rust_helper s64 794 794 + rust_helper_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 795 795 + { 796 796 + return atomic64_fetch_and_relaxed(i, v); 797 797 + } 798 798 + 799 799 + __rust_helper void 800 800 + rust_helper_atomic64_andnot(s64 i, atomic64_t *v) 801 801 + { 802 802 + atomic64_andnot(i, v); 803 803 + } 804 804 + 805 805 + __rust_helper s64 806 806 + rust_helper_atomic64_fetch_andnot(s64 i, atomic64_t *v) 807 807 + { 808 808 + return atomic64_fetch_andnot(i, v); 809 809 + } 810 810 + 811 811 + __rust_helper s64 812 812 + rust_helper_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 813 813 + { 814 814 + return atomic64_fetch_andnot_acquire(i, v); 815 815 + } 816 816 + 817 817 + __rust_helper s64 818 818 + rust_helper_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 819 819 + { 820 820 + return atomic64_fetch_andnot_release(i, v); 821 821 + } 822 822 + 823 823 + __rust_helper s64 824 824 + rust_helper_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 825 825 + { 826 826 + return atomic64_fetch_andnot_relaxed(i, v); 827 827 + } 828 828 + 829 829 + __rust_helper void 830 830 + rust_helper_atomic64_or(s64 i, atomic64_t *v) 831 831 + { 832 832 + atomic64_or(i, v); 833 833 + } 834 834 + 835 835 + __rust_helper s64 836 836 + rust_helper_atomic64_fetch_or(s64 i, atomic64_t *v) 837 837 + { 838 838 + return atomic64_fetch_or(i, v); 839 839 + } 840 840 + 841 841 + __rust_helper s64 842 842 + rust_helper_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 843 843 + { 844 844 + return atomic64_fetch_or_acquire(i, v); 845 845 + } 846 846 + 847 847 + __rust_helper s64 848 848 + rust_helper_atomic64_fetch_or_release(s64 i, atomic64_t *v) 849 849 + { 850 850 + return atomic64_fetch_or_release(i, v); 851 851 + } 852 852 + 853 853 + __rust_helper s64 854 854 + rust_helper_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 855 855 + { 856 856 + return atomic64_fetch_or_relaxed(i, v); 857 857 + } 858 858 + 859 859 + __rust_helper void 860 860 + rust_helper_atomic64_xor(s64 i, atomic64_t *v) 861 861 + { 862 862 + atomic64_xor(i, v); 863 863 + } 864 864 + 865 865 + __rust_helper s64 866 866 + rust_helper_atomic64_fetch_xor(s64 i, atomic64_t *v) 867 867 + { 868 868 + return atomic64_fetch_xor(i, v); 869 869 + } 870 870 + 871 871 + __rust_helper s64 872 872 + rust_helper_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 873 873 + { 874 874 + return atomic64_fetch_xor_acquire(i, v); 875 875 + } 876 876 + 877 877 + __rust_helper s64 878 878 + rust_helper_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 879 879 + { 880 880 + return atomic64_fetch_xor_release(i, v); 881 881 + } 882 882 + 883 883 + __rust_helper s64 884 884 + rust_helper_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 885 885 + { 886 886 + return atomic64_fetch_xor_relaxed(i, v); 887 887 + } 888 888 + 889 889 + __rust_helper s64 890 890 + rust_helper_atomic64_xchg(atomic64_t *v, s64 new) 891 891 + { 892 892 + return atomic64_xchg(v, new); 893 893 + } 894 894 + 895 895 + __rust_helper s64 896 896 + rust_helper_atomic64_xchg_acquire(atomic64_t *v, s64 new) 897 897 + { 898 898 + return atomic64_xchg_acquire(v, new); 899 899 + } 900 900 + 901 901 + __rust_helper s64 902 902 + rust_helper_atomic64_xchg_release(atomic64_t *v, s64 new) 903 903 + { 904 904 + return atomic64_xchg_release(v, new); 905 905 + } 906 906 + 907 907 + __rust_helper s64 908 908 + rust_helper_atomic64_xchg_relaxed(atomic64_t *v, s64 new) 909 909 + { 910 910 + return atomic64_xchg_relaxed(v, new); 911 911 + } 912 912 + 913 913 + __rust_helper s64 914 914 + rust_helper_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 915 915 + { 916 916 + return atomic64_cmpxchg(v, old, new); 917 917 + } 918 918 + 919 919 + __rust_helper s64 920 920 + rust_helper_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 921 921 + { 922 922 + return atomic64_cmpxchg_acquire(v, old, new); 923 923 + } 924 924 + 925 925 + __rust_helper s64 926 926 + rust_helper_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 927 927 + { 928 928 + return atomic64_cmpxchg_release(v, old, new); 929 929 + } 930 930 + 931 931 + __rust_helper s64 932 932 + rust_helper_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 933 933 + { 934 934 + return atomic64_cmpxchg_relaxed(v, old, new); 935 935 + } 936 936 + 937 937 + __rust_helper bool 938 938 + rust_helper_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 939 939 + { 940 940 + return atomic64_try_cmpxchg(v, old, new); 941 941 + } 942 942 + 943 943 + __rust_helper bool 944 944 + rust_helper_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 945 945 + { 946 946 + return atomic64_try_cmpxchg_acquire(v, old, new); 947 947 + } 948 948 + 949 949 + __rust_helper bool 950 950 + rust_helper_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 951 951 + { 952 952 + return atomic64_try_cmpxchg_release(v, old, new); 953 953 + } 954 954 + 955 955 + __rust_helper bool 956 956 + rust_helper_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 957 957 + { 958 958 + return atomic64_try_cmpxchg_relaxed(v, old, new); 959 959 + } 960 960 + 961 961 + __rust_helper bool 962 962 + rust_helper_atomic64_sub_and_test(s64 i, atomic64_t *v) 963 963 + { 964 964 + return atomic64_sub_and_test(i, v); 965 965 + } 966 966 + 967 967 + __rust_helper bool 968 968 + rust_helper_atomic64_dec_and_test(atomic64_t *v) 969 969 + { 970 970 + return atomic64_dec_and_test(v); 971 971 + } 972 972 + 973 973 + __rust_helper bool 974 974 + rust_helper_atomic64_inc_and_test(atomic64_t *v) 975 975 + { 976 976 + return atomic64_inc_and_test(v); 977 977 + } 978 978 + 979 979 + __rust_helper bool 980 980 + rust_helper_atomic64_add_negative(s64 i, atomic64_t *v) 981 981 + { 982 982 + return atomic64_add_negative(i, v); 983 983 + } 984 984 + 985 985 + __rust_helper bool 986 986 + rust_helper_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 987 987 + { 988 988 + return atomic64_add_negative_acquire(i, v); 989 989 + } 990 990 + 991 991 + __rust_helper bool 992 992 + rust_helper_atomic64_add_negative_release(s64 i, atomic64_t *v) 993 993 + { 994 994 + return atomic64_add_negative_release(i, v); 995 995 + } 996 996 + 997 997 + __rust_helper bool 998 998 + rust_helper_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 999 999 + { 1000 1000 + return atomic64_add_negative_relaxed(i, v); 1001 1001 + } 1002 1002 + 1003 1003 + __rust_helper s64 1004 1004 + rust_helper_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 1005 1005 + { 1006 1006 + return atomic64_fetch_add_unless(v, a, u); 1007 1007 + } 1008 1008 + 1009 1009 + __rust_helper bool 1010 1010 + rust_helper_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 1011 1011 + { 1012 1012 + return atomic64_add_unless(v, a, u); 1013 1013 + } 1014 1014 + 1015 1015 + __rust_helper bool 1016 1016 + rust_helper_atomic64_inc_not_zero(atomic64_t *v) 1017 1017 + { 1018 1018 + return atomic64_inc_not_zero(v); 1019 1019 + } 1020 1020 + 1021 1021 + __rust_helper bool 1022 1022 + rust_helper_atomic64_inc_unless_negative(atomic64_t *v) 1023 1023 + { 1024 1024 + return atomic64_inc_unless_negative(v); 1025 1025 + } 1026 1026 + 1027 1027 + __rust_helper bool 1028 1028 + rust_helper_atomic64_dec_unless_positive(atomic64_t *v) 1029 1029 + { 1030 1030 + return atomic64_dec_unless_positive(v); 1031 1031 + } 1032 1032 + 1033 1033 + __rust_helper s64 1034 1034 + rust_helper_atomic64_dec_if_positive(atomic64_t *v) 1035 1035 + { 1036 1036 + return atomic64_dec_if_positive(v); 1037 1037 + } 1038 1038 + 1039 1039 + #endif /* _RUST_ATOMIC_API_H */ 1040 1040 + // 615a0e0c98b5973a47fe4fa65e92935051ca00ed

+18

rust/helpers/barrier.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + #include <asm/barrier.h> 4 4 + 5 5 + void rust_helper_smp_mb(void) 6 6 + { 7 7 + smp_mb(); 8 8 + } 9 9 + 10 10 + void rust_helper_smp_wmb(void) 11 11 + { 12 12 + smp_wmb(); 13 13 + } 14 14 + 15 15 + void rust_helper_smp_rmb(void) 16 16 + { 17 17 + smp_rmb(); 18 18 + }

+2

rust/helpers/helpers.c

reviewed

··· 7 7 * Sorted alphabetically. 8 8 */ 9 9 10 10 + #include "atomic.c" 10 11 #include "auxiliary.c" 12 12 + #include "barrier.c" 11 13 #include "blk.c" 12 14 #include "bug.c" 13 15 #include "build_assert.c"

+10

rust/helpers/refcount.c

reviewed

··· 7 7 return (refcount_t)REFCOUNT_INIT(n); 8 8 } 9 9 10 10 + void rust_helper_refcount_set(refcount_t *r, int n) 11 11 + { 12 12 + refcount_set(r, n); 13 13 + } 14 14 + 10 15 void rust_helper_refcount_inc(refcount_t *r) 11 16 { 12 17 refcount_inc(r); 18 18 + } 19 19 + 20 20 + void rust_helper_refcount_dec(refcount_t *r) 21 21 + { 22 22 + refcount_dec(r); 13 23 } 14 24 15 25 bool rust_helper_refcount_dec_and_test(refcount_t *r)

+4 -3

rust/kernel/block/mq/operations.rs

reviewed

··· 10 10 block::mq::Request, 11 11 error::{from_result, Result}, 12 12 prelude::*, 13 13 + sync::Refcount, 13 14 types::ARef, 14 15 }; 15 15 - use core::{marker::PhantomData, sync::atomic::AtomicU64, sync::atomic::Ordering}; 16 16 + use core::marker::PhantomData; 16 17 17 18 /// Implement this trait to interface blk-mq as block devices. 18 19 /// ··· 79 78 let request = unsafe { &*(*bd).rq.cast::<Request<T>>() }; 80 79 81 80 // One refcount for the ARef, one for being in flight 82 82 - request.wrapper_ref().refcount().store(2, Ordering::Relaxed); 81 81 + request.wrapper_ref().refcount().set(2); 83 82 84 83 // SAFETY: 85 84 // - We own a refcount that we took above. We pass that to `ARef`. ··· 188 187 189 188 // SAFETY: The refcount field is allocated but not initialized, so 190 189 // it is valid for writes. 191 191 - unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(AtomicU64::new(0)) }; 190 190 + unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; 192 191 193 192 Ok(0) 194 193 })

+21 -52

rust/kernel/block/mq/request.rs

reviewed

··· 8 8 bindings, 9 9 block::mq::Operations, 10 10 error::Result, 11 11 + sync::{atomic::Relaxed, Refcount}, 11 12 types::{ARef, AlwaysRefCounted, Opaque}, 12 13 }; 13 13 - use core::{ 14 14 - marker::PhantomData, 15 15 - ptr::NonNull, 16 16 - sync::atomic::{AtomicU64, Ordering}, 17 17 - }; 14 14 + use core::{marker::PhantomData, ptr::NonNull}; 18 15 19 16 /// A wrapper around a blk-mq [`struct request`]. This represents an IO request. 20 17 /// ··· 33 36 /// 34 37 /// We need to track 3 and 4 to ensure that it is safe to end the request and hand 35 38 /// back ownership to the block layer. 39 39 + /// 40 40 + /// Note that the driver can still obtain new `ARef` even if there is no `ARef`s in existence by 41 41 + /// using `tag_to_rq`, hence the need to distinguish B and C. 36 42 /// 37 43 /// The states are tracked through the private `refcount` field of 38 44 /// `RequestDataWrapper`. This structure lives in the private data area of the C ··· 98 98 /// 99 99 /// [`struct request`]: srctree/include/linux/blk-mq.h 100 100 fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> { 101 101 - // We can race with `TagSet::tag_to_rq` 102 102 - if let Err(_old) = this.wrapper_ref().refcount().compare_exchange( 103 103 - 2, 104 104 - 0, 105 105 - Ordering::Relaxed, 106 106 - Ordering::Relaxed, 107 107 - ) { 101 101 + // To hand back the ownership, we need the current refcount to be 2. 102 102 + // Since we can race with `TagSet::tag_to_rq`, this needs to atomically reduce 103 103 + // refcount to 0. `Refcount` does not provide a way to do this, so use the underlying 104 104 + // atomics directly. 105 105 + if let Err(_old) = this 106 106 + .wrapper_ref() 107 107 + .refcount() 108 108 + .as_atomic() 109 109 + .cmpxchg(2, 0, Relaxed) 110 110 + { 108 111 return Err(this); 109 112 } 110 113 ··· 176 173 /// - 0: The request is owned by C block layer. 177 174 /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. 178 175 /// - 2+: There are [`ARef`] references to the request. 179 179 - refcount: AtomicU64, 176 176 + refcount: Refcount, 180 177 } 181 178 182 179 impl RequestDataWrapper { 183 180 /// Return a reference to the refcount of the request that is embedding 184 181 /// `self`. 185 185 - pub(crate) fn refcount(&self) -> &AtomicU64 { 182 182 + pub(crate) fn refcount(&self) -> &Refcount { 186 183 &self.refcount 187 184 } 188 185 ··· 192 189 /// # Safety 193 190 /// 194 191 /// - `this` must point to a live allocation of at least the size of `Self`. 195 195 - pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut AtomicU64 { 192 192 + pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { 196 193 // SAFETY: Because of the safety requirements of this function, the 197 194 // field projection is safe. 198 195 unsafe { &raw mut (*this).refcount } ··· 208 205 // mutate `self` are internally synchronized` 209 206 unsafe impl<T: Operations> Sync for Request<T> {} 210 207 211 211 - /// Store the result of `op(target.load())` in target, returning new value of 212 212 - /// target. 213 213 - fn atomic_relaxed_op_return(target: &AtomicU64, op: impl Fn(u64) -> u64) -> u64 { 214 214 - let old = target.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |x| Some(op(x))); 215 215 - 216 216 - // SAFETY: Because the operation passed to `fetch_update` above always 217 217 - // return `Some`, `old` will always be `Ok`. 218 218 - let old = unsafe { old.unwrap_unchecked() }; 219 219 - 220 220 - op(old) 221 221 - } 222 222 - 223 223 - /// Store the result of `op(target.load)` in `target` if `target.load() != 224 224 - /// pred`, returning [`true`] if the target was updated. 225 225 - fn atomic_relaxed_op_unless(target: &AtomicU64, op: impl Fn(u64) -> u64, pred: u64) -> bool { 226 226 - target 227 227 - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |x| { 228 228 - if x == pred { 229 229 - None 230 230 - } else { 231 231 - Some(op(x)) 232 232 - } 233 233 - }) 234 234 - .is_ok() 235 235 - } 236 236 - 237 208 // SAFETY: All instances of `Request<T>` are reference counted. This 238 209 // implementation of `AlwaysRefCounted` ensure that increments to the ref count 239 210 // keeps the object alive in memory at least until a matching reference count 240 211 // decrement is executed. 241 212 unsafe impl<T: Operations> AlwaysRefCounted for Request<T> { 242 213 fn inc_ref(&self) { 243 243 - let refcount = &self.wrapper_ref().refcount(); 244 244 - 245 245 - #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))] 246 246 - let updated = atomic_relaxed_op_unless(refcount, |x| x + 1, 0); 247 247 - 248 248 - #[cfg(CONFIG_DEBUG_MISC)] 249 249 - if !updated { 250 250 - panic!("Request refcount zero on clone") 251 251 - } 214 214 + self.wrapper_ref().refcount().inc(); 252 215 } 253 216 254 217 unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) { ··· 226 257 let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) }; 227 258 228 259 #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))] 229 229 - let new_refcount = atomic_relaxed_op_return(refcount, |x| x - 1); 260 260 + let is_zero = refcount.dec_and_test(); 230 261 231 262 #[cfg(CONFIG_DEBUG_MISC)] 232 232 - if new_refcount == 0 { 263 263 + if is_zero { 233 264 panic!("Request reached refcount zero in Rust abstractions"); 234 265 } 235 266 }

+4

rust/kernel/sync.rs

reviewed

··· 11 11 12 12 mod arc; 13 13 pub mod aref; 14 14 + pub mod atomic; 15 15 + pub mod barrier; 14 16 pub mod completion; 15 17 mod condvar; 16 18 pub mod lock; 17 19 mod locked_by; 18 20 pub mod poll; 19 21 pub mod rcu; 22 22 + mod refcount; 20 23 21 24 pub use arc::{Arc, ArcBorrow, UniqueArc}; 22 25 pub use completion::Completion; ··· 28 25 pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; 29 26 pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; 30 27 pub use locked_by::LockedBy; 28 28 + pub use refcount::Refcount; 31 29 32 30 /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. 33 31 #[repr(transparent)]

+19 -36

rust/kernel/sync/arc.rs

reviewed

··· 8 8 //! threads. 9 9 //! 10 10 //! It is different from the standard library's [`Arc`] in a few ways: 11 11 - //! 1. It is backed by the kernel's `refcount_t` type. 11 11 + //! 1. It is backed by the kernel's [`Refcount`] type. 12 12 //! 2. It does not support weak references, which allows it to be half the size. 13 13 //! 3. It saturates the reference count instead of aborting when it goes over a threshold. 14 14 //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. ··· 18 18 19 19 use crate::{ 20 20 alloc::{AllocError, Flags, KBox}, 21 21 - bindings, 22 21 ffi::c_void, 23 22 init::InPlaceInit, 23 23 + sync::Refcount, 24 24 try_init, 25 25 - types::{ForeignOwnable, Opaque}, 25 25 + types::ForeignOwnable, 26 26 }; 27 27 use core::{ 28 28 alloc::Layout, ··· 145 145 #[pin_data] 146 146 #[repr(C)] 147 147 struct ArcInner<T: ?Sized> { 148 148 - refcount: Opaque<bindings::refcount_t>, 148 148 + refcount: Refcount, 149 149 data: T, 150 150 } 151 151 ··· 157 157 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must 158 158 /// not yet have been destroyed. 159 159 unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { 160 160 - let refcount_layout = Layout::new::<bindings::refcount_t>(); 160 160 + let refcount_layout = Layout::new::<Refcount>(); 161 161 // SAFETY: The caller guarantees that the pointer is valid. 162 162 let val_layout = Layout::for_value(unsafe { &*ptr }); 163 163 // SAFETY: We're computing the layout of a real struct that existed when compiling this ··· 229 229 pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { 230 230 // INVARIANT: The refcount is initialised to a non-zero value. 231 231 let value = ArcInner { 232 232 - // SAFETY: There are no safety requirements for this FFI call. 233 233 - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), 232 232 + refcount: Refcount::new(1), 234 233 data: contents, 235 234 }; 236 235 ··· 320 321 /// use kernel::sync::{Arc, UniqueArc}; 321 322 /// 322 323 /// let arc = Arc::new(42, GFP_KERNEL)?; 323 323 - /// let unique_arc = arc.into_unique_or_drop(); 324 324 + /// let unique_arc = Arc::into_unique_or_drop(arc); 324 325 /// 325 326 /// // The above conversion should succeed since refcount of `arc` is 1. 326 327 /// assert!(unique_arc.is_some()); ··· 336 337 /// let arc = Arc::new(42, GFP_KERNEL)?; 337 338 /// let another = arc.clone(); 338 339 /// 339 339 - /// let unique_arc = arc.into_unique_or_drop(); 340 340 + /// let unique_arc = Arc::into_unique_or_drop(arc); 340 341 /// 341 342 /// // The above conversion should fail since refcount of `arc` is >1. 342 343 /// assert!(unique_arc.is_none()); 343 344 /// 344 345 /// # Ok::<(), Error>(()) 345 346 /// ``` 346 346 - pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { 347 347 + pub fn into_unique_or_drop(this: Self) -> Option<Pin<UniqueArc<T>>> { 347 348 // We will manually manage the refcount in this method, so we disable the destructor. 348 348 - let me = ManuallyDrop::new(self); 349 349 + let this = ManuallyDrop::new(this); 349 350 // SAFETY: We own a refcount, so the pointer is still valid. 350 350 - let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); 351 351 + let refcount = unsafe { &this.ptr.as_ref().refcount }; 351 352 352 353 // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will 353 354 // return without further touching the `Arc`. If the refcount reaches zero, then there are 354 355 // no other arcs, and we can create a `UniqueArc`. 355 355 - // 356 356 - // SAFETY: We own a refcount, so the pointer is not dangling. 357 357 - let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; 358 358 - if is_zero { 359 359 - // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized 360 360 - // accesses to the refcount. 361 361 - unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; 356 356 + if refcount.dec_and_test() { 357 357 + refcount.set(1); 362 358 363 359 // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We 364 360 // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin 365 361 // their values. 366 362 Some(Pin::from(UniqueArc { 367 367 - inner: ManuallyDrop::into_inner(me), 363 363 + inner: ManuallyDrop::into_inner(this), 368 364 })) 369 365 } else { 370 366 None ··· 450 456 451 457 impl<T: ?Sized> Clone for Arc<T> { 452 458 fn clone(&self) -> Self { 453 453 - // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is 454 454 - // safe to dereference it. 455 455 - let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); 456 456 - 457 457 - // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. 459 459 + // INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero. 458 460 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is 459 461 // safe to increment the refcount. 460 460 - unsafe { bindings::refcount_inc(refcount) }; 462 462 + unsafe { self.ptr.as_ref() }.refcount.inc(); 461 463 462 464 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. 463 465 unsafe { Self::from_inner(self.ptr) } ··· 462 472 463 473 impl<T: ?Sized> Drop for Arc<T> { 464 474 fn drop(&mut self) { 465 465 - // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot 466 466 - // touch `refcount` after it's decremented to a non-zero value because another thread/CPU 467 467 - // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to 468 468 - // freed/invalid memory as long as it is never dereferenced. 469 469 - let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); 470 470 - 471 475 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and 472 476 // this instance is being dropped, so the broken invariant is not observable. 473 473 - // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. 474 474 - let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; 477 477 + // SAFETY: By the type invariant, there is necessarily a reference to the object. 478 478 + let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test(); 475 479 if is_zero { 476 480 // The count reached zero, we must free the memory. 477 481 // ··· 759 775 // INVARIANT: The refcount is initialised to a non-zero value. 760 776 let inner = KBox::try_init::<AllocError>( 761 777 try_init!(ArcInner { 762 762 - // SAFETY: There are no safety requirements for this FFI call. 763 763 - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), 778 778 + refcount: Refcount::new(1), 764 779 data <- pin_init::uninit::<T, AllocError>(), 765 780 }? AllocError), 766 781 flags,

+551

rust/kernel/sync/atomic.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Atomic primitives. 4 4 + //! 5 5 + //! These primitives have the same semantics as their C counterparts: and the precise definitions of 6 6 + //! semantics can be found at [`LKMM`]. Note that Linux Kernel Memory (Consistency) Model is the 7 7 + //! only model for Rust code in kernel, and Rust's own atomics should be avoided. 8 8 + //! 9 9 + //! # Data races 10 10 + //! 11 11 + //! [`LKMM`] atomics have different rules regarding data races: 12 12 + //! 13 13 + //! - A normal write from C side is treated as an atomic write if 14 14 + //! CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=y. 15 15 + //! - Mixed-size atomic accesses don't cause data races. 16 16 + //! 17 17 + //! [`LKMM`]: srctree/tools/memory-model/ 18 18 + 19 19 + mod internal; 20 20 + pub mod ordering; 21 21 + mod predefine; 22 22 + 23 23 + pub use internal::AtomicImpl; 24 24 + pub use ordering::{Acquire, Full, Relaxed, Release}; 25 25 + 26 26 + use crate::build_error; 27 27 + use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps, AtomicRepr}; 28 28 + use ordering::OrderingType; 29 29 + 30 30 + /// A memory location which can be safely modified from multiple execution contexts. 31 31 + /// 32 32 + /// This has the same size, alignment and bit validity as the underlying type `T`. And it disables 33 33 + /// niche optimization for the same reason as [`UnsafeCell`]. 34 34 + /// 35 35 + /// The atomic operations are implemented in a way that is fully compatible with the [Linux Kernel 36 36 + /// Memory (Consistency) Model][LKMM], hence they should be modeled as the corresponding 37 37 + /// [`LKMM`][LKMM] atomic primitives. With the help of [`Atomic::from_ptr()`] and 38 38 + /// [`Atomic::as_ptr()`], this provides a way to interact with [C-side atomic operations] 39 39 + /// (including those without the `atomic` prefix, e.g. `READ_ONCE()`, `WRITE_ONCE()`, 40 40 + /// `smp_load_acquire()` and `smp_store_release()`). 41 41 + /// 42 42 + /// # Invariants 43 43 + /// 44 44 + /// `self.0` is a valid `T`. 45 45 + /// 46 46 + /// [`UnsafeCell`]: core::cell::UnsafeCell 47 47 + /// [LKMM]: srctree/tools/memory-model/ 48 48 + /// [C-side atomic operations]: srctree/Documentation/atomic_t.txt 49 49 + #[repr(transparent)] 50 50 + pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>); 51 51 + 52 52 + // SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic. 53 53 + unsafe impl<T: AtomicType> Sync for Atomic<T> {} 54 54 + 55 55 + /// Types that support basic atomic operations. 56 56 + /// 57 57 + /// # Round-trip transmutability 58 58 + /// 59 59 + /// `T` is round-trip transmutable to `U` if and only if both of these properties hold: 60 60 + /// 61 61 + /// - Any valid bit pattern for `T` is also a valid bit pattern for `U`. 62 62 + /// - Transmuting (e.g. using [`transmute()`]) a value of type `T` to `U` and then to `T` again 63 63 + /// yields a value that is in all aspects equivalent to the original value. 64 64 + /// 65 65 + /// # Safety 66 66 + /// 67 67 + /// - [`Self`] must have the same size and alignment as [`Self::Repr`]. 68 68 + /// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. 69 69 + /// 70 70 + /// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. 71 71 + /// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic 72 72 + /// variables over unit-only enums, see [Examples]. 73 73 + /// 74 74 + /// # Limitations 75 75 + /// 76 76 + /// Because C primitives are used to implement the atomic operations, and a C function requires a 77 77 + /// valid object of a type to operate on (i.e. no `MaybeUninit<_>`), hence at the Rust <-> C 78 78 + /// surface, only types with all the bits initialized can be passed. As a result, types like `(u8, 79 79 + /// u16)` (padding bytes are uninitialized) are currently not supported. 80 80 + /// 81 81 + /// # Examples 82 82 + /// 83 83 + /// A unit-only enum that implements [`AtomicType`]: 84 84 + /// 85 85 + /// ``` 86 86 + /// use kernel::sync::atomic::{AtomicType, Atomic, Relaxed}; 87 87 + /// 88 88 + /// #[derive(Clone, Copy, PartialEq, Eq)] 89 89 + /// #[repr(i32)] 90 90 + /// enum State { 91 91 + /// Uninit = 0, 92 92 + /// Working = 1, 93 93 + /// Done = 2, 94 94 + /// }; 95 95 + /// 96 96 + /// // SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip 97 97 + /// // transmutable to `i32`. 98 98 + /// unsafe impl AtomicType for State { 99 99 + /// type Repr = i32; 100 100 + /// } 101 101 + /// 102 102 + /// let s = Atomic::new(State::Uninit); 103 103 + /// 104 104 + /// assert_eq!(State::Uninit, s.load(Relaxed)); 105 105 + /// ``` 106 106 + /// [`transmute()`]: core::mem::transmute 107 107 + /// [round-trip transmutable]: AtomicType#round-trip-transmutability 108 108 + /// [Examples]: AtomicType#examples 109 109 + pub unsafe trait AtomicType: Sized + Send + Copy { 110 110 + /// The backing atomic implementation type. 111 111 + type Repr: AtomicImpl; 112 112 + } 113 113 + 114 114 + /// Types that support atomic add operations. 115 115 + /// 116 116 + /// # Safety 117 117 + /// 118 118 + // TODO: Properly defines `wrapping_add` in the following comment. 119 119 + /// `wrapping_add` any value of type `Self::Repr::Delta` obtained by [`Self::rhs_into_delta()`] to 120 120 + /// any value of type `Self::Repr` obtained through transmuting a value of type `Self` to must 121 121 + /// yield a value with a bit pattern also valid for `Self`. 122 122 + pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType { 123 123 + /// Converts `Rhs` into the `Delta` type of the atomic implementation. 124 124 + fn rhs_into_delta(rhs: Rhs) -> <Self::Repr as AtomicImpl>::Delta; 125 125 + } 126 126 + 127 127 + #[inline(always)] 128 128 + const fn into_repr<T: AtomicType>(v: T) -> T::Repr { 129 129 + // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to 130 130 + // `T::Repr`, therefore the transmute operation is sound. 131 131 + unsafe { core::mem::transmute_copy(&v) } 132 132 + } 133 133 + 134 134 + /// # Safety 135 135 + /// 136 136 + /// `r` must be a valid bit pattern of `T`. 137 137 + #[inline(always)] 138 138 + const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T { 139 139 + // SAFETY: Per the safety requirement of the function, the transmute operation is sound. 140 140 + unsafe { core::mem::transmute_copy(&r) } 141 141 + } 142 142 + 143 143 + impl<T: AtomicType> Atomic<T> { 144 144 + /// Creates a new atomic `T`. 145 145 + pub const fn new(v: T) -> Self { 146 146 + // INVARIANT: Per the safety requirement of `AtomicType`, `into_repr(v)` is a valid `T`. 147 147 + Self(AtomicRepr::new(into_repr(v))) 148 148 + } 149 149 + 150 150 + /// Creates a reference to an atomic `T` from a pointer of `T`. 151 151 + /// 152 152 + /// This usually is used when communicating with C side or manipulating a C struct, see 153 153 + /// examples below. 154 154 + /// 155 155 + /// # Safety 156 156 + /// 157 157 + /// - `ptr` is aligned to `align_of::<T>()`. 158 158 + /// - `ptr` is valid for reads and writes for `'a`. 159 159 + /// - For the duration of `'a`, other accesses to `*ptr` must not cause data races (defined 160 160 + /// by [`LKMM`]) against atomic operations on the returned reference. Note that if all other 161 161 + /// accesses are atomic, then this safety requirement is trivially fulfilled. 162 162 + /// 163 163 + /// [`LKMM`]: srctree/tools/memory-model 164 164 + /// 165 165 + /// # Examples 166 166 + /// 167 167 + /// Using [`Atomic::from_ptr()`] combined with [`Atomic::load()`] or [`Atomic::store()`] can 168 168 + /// achieve the same functionality as `READ_ONCE()`/`smp_load_acquire()` or 169 169 + /// `WRITE_ONCE()`/`smp_store_release()` in C side: 170 170 + /// 171 171 + /// ``` 172 172 + /// # use kernel::types::Opaque; 173 173 + /// use kernel::sync::atomic::{Atomic, Relaxed, Release}; 174 174 + /// 175 175 + /// // Assume there is a C struct `foo`. 176 176 + /// mod cbindings { 177 177 + /// #[repr(C)] 178 178 + /// pub(crate) struct foo { 179 179 + /// pub(crate) a: i32, 180 180 + /// pub(crate) b: i32 181 181 + /// } 182 182 + /// } 183 183 + /// 184 184 + /// let tmp = Opaque::new(cbindings::foo { a: 1, b: 2 }); 185 185 + /// 186 186 + /// // struct foo *foo_ptr = ..; 187 187 + /// let foo_ptr = tmp.get(); 188 188 + /// 189 189 + /// // SAFETY: `foo_ptr` is valid, and `.a` is in bounds. 190 190 + /// let foo_a_ptr = unsafe { &raw mut (*foo_ptr).a }; 191 191 + /// 192 192 + /// // a = READ_ONCE(foo_ptr->a); 193 193 + /// // 194 194 + /// // SAFETY: `foo_a_ptr` is valid for read, and all other accesses on it is atomic, so no 195 195 + /// // data race. 196 196 + /// let a = unsafe { Atomic::from_ptr(foo_a_ptr) }.load(Relaxed); 197 197 + /// # assert_eq!(a, 1); 198 198 + /// 199 199 + /// // smp_store_release(&foo_ptr->a, 2); 200 200 + /// // 201 201 + /// // SAFETY: `foo_a_ptr` is valid for writes, and all other accesses on it is atomic, so 202 202 + /// // no data race. 203 203 + /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); 204 204 + /// ``` 205 205 + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self 206 206 + where 207 207 + T: Sync, 208 208 + { 209 209 + // CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity. 210 210 + // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will 211 211 + // live long enough. It's safe to return a `&Atomic<T>` because function safety requirement 212 212 + // guarantees other accesses won't cause data races. 213 213 + unsafe { &*ptr.cast::<Self>() } 214 214 + } 215 215 + 216 216 + /// Returns a pointer to the underlying atomic `T`. 217 217 + /// 218 218 + /// Note that use of the return pointer must not cause data races defined by [`LKMM`]. 219 219 + /// 220 220 + /// # Guarantees 221 221 + /// 222 222 + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]). 223 223 + /// 224 224 + /// [`LKMM`]: srctree/tools/memory-model 225 225 + /// [`align_of::<T>()`]: core::mem::align_of 226 226 + pub const fn as_ptr(&self) -> *mut T { 227 227 + // GUARANTEE: Per the function guarantee of `AtomicRepr::as_ptr()`, the `self.0.as_ptr()` 228 228 + // must be a valid and properly aligned pointer for `T::Repr`, and per the safety guarantee 229 229 + // of `AtomicType`, it's a valid and properly aligned pointer of `T`. 230 230 + self.0.as_ptr().cast() 231 231 + } 232 232 + 233 233 + /// Returns a mutable reference to the underlying atomic `T`. 234 234 + /// 235 235 + /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. 236 236 + pub fn get_mut(&mut self) -> &mut T { 237 237 + // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of 238 238 + // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting 239 239 + // result is a valid pointer of `T`. 240 240 + // SAFETY: The pointer is valid per the CAST comment above, and the mutable reference 241 241 + // guarantees exclusive access. 242 242 + unsafe { &mut *self.0.as_ptr().cast() } 243 243 + } 244 244 + } 245 245 + 246 246 + impl<T: AtomicType> Atomic<T> 247 247 + where 248 248 + T::Repr: AtomicBasicOps, 249 249 + { 250 250 + /// Loads the value from the atomic `T`. 251 251 + /// 252 252 + /// # Examples 253 253 + /// 254 254 + /// ``` 255 255 + /// use kernel::sync::atomic::{Atomic, Relaxed}; 256 256 + /// 257 257 + /// let x = Atomic::new(42i32); 258 258 + /// 259 259 + /// assert_eq!(42, x.load(Relaxed)); 260 260 + /// 261 261 + /// let x = Atomic::new(42i64); 262 262 + /// 263 263 + /// assert_eq!(42, x.load(Relaxed)); 264 264 + /// ``` 265 265 + #[doc(alias("atomic_read", "atomic64_read"))] 266 266 + #[inline(always)] 267 267 + pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, _: Ordering) -> T { 268 268 + let v = { 269 269 + match Ordering::TYPE { 270 270 + OrderingType::Relaxed => T::Repr::atomic_read(&self.0), 271 271 + OrderingType::Acquire => T::Repr::atomic_read_acquire(&self.0), 272 272 + _ => build_error!("Wrong ordering"), 273 273 + } 274 274 + }; 275 275 + 276 276 + // SAFETY: `v` comes from reading `self.0`, which is a valid `T` per the type invariants. 277 277 + unsafe { from_repr(v) } 278 278 + } 279 279 + 280 280 + /// Stores a value to the atomic `T`. 281 281 + /// 282 282 + /// # Examples 283 283 + /// 284 284 + /// ``` 285 285 + /// use kernel::sync::atomic::{Atomic, Relaxed}; 286 286 + /// 287 287 + /// let x = Atomic::new(42i32); 288 288 + /// 289 289 + /// assert_eq!(42, x.load(Relaxed)); 290 290 + /// 291 291 + /// x.store(43, Relaxed); 292 292 + /// 293 293 + /// assert_eq!(43, x.load(Relaxed)); 294 294 + /// ``` 295 295 + #[doc(alias("atomic_set", "atomic64_set"))] 296 296 + #[inline(always)] 297 297 + pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: T, _: Ordering) { 298 298 + let v = into_repr(v); 299 299 + 300 300 + // INVARIANT: `v` is a valid `T`, and is stored to `self.0` by `atomic_set*()`. 301 301 + match Ordering::TYPE { 302 302 + OrderingType::Relaxed => T::Repr::atomic_set(&self.0, v), 303 303 + OrderingType::Release => T::Repr::atomic_set_release(&self.0, v), 304 304 + _ => build_error!("Wrong ordering"), 305 305 + } 306 306 + } 307 307 + } 308 308 + 309 309 + impl<T: AtomicType> Atomic<T> 310 310 + where 311 311 + T::Repr: AtomicExchangeOps, 312 312 + { 313 313 + /// Atomic exchange. 314 314 + /// 315 315 + /// Atomically updates `*self` to `v` and returns the old value of `*self`. 316 316 + /// 317 317 + /// # Examples 318 318 + /// 319 319 + /// ``` 320 320 + /// use kernel::sync::atomic::{Atomic, Acquire, Relaxed}; 321 321 + /// 322 322 + /// let x = Atomic::new(42); 323 323 + /// 324 324 + /// assert_eq!(42, x.xchg(52, Acquire)); 325 325 + /// assert_eq!(52, x.load(Relaxed)); 326 326 + /// ``` 327 327 + #[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))] 328 328 + #[inline(always)] 329 329 + pub fn xchg<Ordering: ordering::Ordering>(&self, v: T, _: Ordering) -> T { 330 330 + let v = into_repr(v); 331 331 + 332 332 + // INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to 333 333 + // `T`. 334 334 + let ret = { 335 335 + match Ordering::TYPE { 336 336 + OrderingType::Full => T::Repr::atomic_xchg(&self.0, v), 337 337 + OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v), 338 338 + OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v), 339 339 + OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v), 340 340 + } 341 341 + }; 342 342 + 343 343 + // SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants. 344 344 + unsafe { from_repr(ret) } 345 345 + } 346 346 + 347 347 + /// Atomic compare and exchange. 348 348 + /// 349 349 + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not 350 350 + /// modified. 351 351 + /// 352 352 + /// Compare: The comparison is done via the byte level comparison between `*self` and `old`. 353 353 + /// 354 354 + /// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type 355 355 + /// parameter indicates, and a failed one doesn't provide any ordering, the load part of a 356 356 + /// failed cmpxchg is a [`Relaxed`] load. 357 357 + /// 358 358 + /// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`, 359 359 + /// otherwise returns `Err(value)`, and `value` is the current value of `*self`. 360 360 + /// 361 361 + /// # Examples 362 362 + /// 363 363 + /// ``` 364 364 + /// use kernel::sync::atomic::{Atomic, Full, Relaxed}; 365 365 + /// 366 366 + /// let x = Atomic::new(42); 367 367 + /// 368 368 + /// // Checks whether cmpxchg succeeded. 369 369 + /// let success = x.cmpxchg(52, 64, Relaxed).is_ok(); 370 370 + /// # assert!(!success); 371 371 + /// 372 372 + /// // Checks whether cmpxchg failed. 373 373 + /// let failure = x.cmpxchg(52, 64, Relaxed).is_err(); 374 374 + /// # assert!(failure); 375 375 + /// 376 376 + /// // Uses the old value if failed, probably re-try cmpxchg. 377 377 + /// match x.cmpxchg(52, 64, Relaxed) { 378 378 + /// Ok(_) => { }, 379 379 + /// Err(old) => { 380 380 + /// // do something with `old`. 381 381 + /// # assert_eq!(old, 42); 382 382 + /// } 383 383 + /// } 384 384 + /// 385 385 + /// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C. 386 386 + /// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); 387 387 + /// # assert_eq!(42, latest); 388 388 + /// assert_eq!(64, x.load(Relaxed)); 389 389 + /// ``` 390 390 + /// 391 391 + /// [`Relaxed`]: ordering::Relaxed 392 392 + #[doc(alias( 393 393 + "atomic_cmpxchg", 394 394 + "atomic64_cmpxchg", 395 395 + "atomic_try_cmpxchg", 396 396 + "atomic64_try_cmpxchg", 397 397 + "compare_exchange" 398 398 + ))] 399 399 + #[inline(always)] 400 400 + pub fn cmpxchg<Ordering: ordering::Ordering>( 401 401 + &self, 402 402 + mut old: T, 403 403 + new: T, 404 404 + o: Ordering, 405 405 + ) -> Result<T, T> { 406 406 + // Note on code generation: 407 407 + // 408 408 + // try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined, 409 409 + // the compiler is able to figure out that branch is not needed if the users don't care 410 410 + // about whether the operation succeeds or not. One exception is on x86, due to commit 411 411 + // 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the 412 412 + // atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the 413 413 + // success of cmpxchg and only wants to use the old value. For example, for code like: 414 414 + // 415 415 + // let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); 416 416 + // 417 417 + // It will still generate code: 418 418 + // 419 419 + // movl $0x40, %ecx 420 420 + // movl $0x34, %eax 421 421 + // lock 422 422 + // cmpxchgl %ecx, 0x4(%rsp) 423 423 + // jne 1f 424 424 + // 2: 425 425 + // ... 426 426 + // 1: movl %eax, %ecx 427 427 + // jmp 2b 428 428 + // 429 429 + // This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old" 430 430 + // location in the C function is always safe to write. 431 431 + if self.try_cmpxchg(&mut old, new, o) { 432 432 + Ok(old) 433 433 + } else { 434 434 + Err(old) 435 435 + } 436 436 + } 437 437 + 438 438 + /// Atomic compare and exchange and returns whether the operation succeeds. 439 439 + /// 440 440 + /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not 441 441 + /// modified, `*old` is updated to the current value of `*self`. 442 442 + /// 443 443 + /// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`]. 444 444 + /// 445 445 + /// Returns `true` means the cmpxchg succeeds otherwise returns `false`. 446 446 + #[inline(always)] 447 447 + fn try_cmpxchg<Ordering: ordering::Ordering>(&self, old: &mut T, new: T, _: Ordering) -> bool { 448 448 + let mut tmp = into_repr(*old); 449 449 + let new = into_repr(new); 450 450 + 451 451 + // INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is 452 452 + // transmutable to `T`. 453 453 + let ret = { 454 454 + match Ordering::TYPE { 455 455 + OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new), 456 456 + OrderingType::Acquire => { 457 457 + T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new) 458 458 + } 459 459 + OrderingType::Release => { 460 460 + T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new) 461 461 + } 462 462 + OrderingType::Relaxed => { 463 463 + T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new) 464 464 + } 465 465 + } 466 466 + }; 467 467 + 468 468 + // SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants. 469 469 + *old = unsafe { from_repr(tmp) }; 470 470 + 471 471 + ret 472 472 + } 473 473 + } 474 474 + 475 475 + impl<T: AtomicType> Atomic<T> 476 476 + where 477 477 + T::Repr: AtomicArithmeticOps, 478 478 + { 479 479 + /// Atomic add. 480 480 + /// 481 481 + /// Atomically updates `*self` to `(*self).wrapping_add(v)`. 482 482 + /// 483 483 + /// # Examples 484 484 + /// 485 485 + /// ``` 486 486 + /// use kernel::sync::atomic::{Atomic, Relaxed}; 487 487 + /// 488 488 + /// let x = Atomic::new(42); 489 489 + /// 490 490 + /// assert_eq!(42, x.load(Relaxed)); 491 491 + /// 492 492 + /// x.add(12, Relaxed); 493 493 + /// 494 494 + /// assert_eq!(54, x.load(Relaxed)); 495 495 + /// ``` 496 496 + #[inline(always)] 497 497 + pub fn add<Rhs>(&self, v: Rhs, _: ordering::Relaxed) 498 498 + where 499 499 + T: AtomicAdd<Rhs>, 500 500 + { 501 501 + let v = T::rhs_into_delta(v); 502 502 + 503 503 + // INVARIANT: `self.0` is a valid `T` after `atomic_add()` due to safety requirement of 504 504 + // `AtomicAdd`. 505 505 + T::Repr::atomic_add(&self.0, v); 506 506 + } 507 507 + 508 508 + /// Atomic fetch and add. 509 509 + /// 510 510 + /// Atomically updates `*self` to `(*self).wrapping_add(v)`, and returns the value of `*self` 511 511 + /// before the update. 512 512 + /// 513 513 + /// # Examples 514 514 + /// 515 515 + /// ``` 516 516 + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; 517 517 + /// 518 518 + /// let x = Atomic::new(42); 519 519 + /// 520 520 + /// assert_eq!(42, x.load(Relaxed)); 521 521 + /// 522 522 + /// assert_eq!(54, { x.fetch_add(12, Acquire); x.load(Relaxed) }); 523 523 + /// 524 524 + /// let x = Atomic::new(42); 525 525 + /// 526 526 + /// assert_eq!(42, x.load(Relaxed)); 527 527 + /// 528 528 + /// assert_eq!(54, { x.fetch_add(12, Full); x.load(Relaxed) } ); 529 529 + /// ``` 530 530 + #[inline(always)] 531 531 + pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T 532 532 + where 533 533 + T: AtomicAdd<Rhs>, 534 534 + { 535 535 + let v = T::rhs_into_delta(v); 536 536 + 537 537 + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_add*()` due to safety requirement 538 538 + // of `AtomicAdd`. 539 539 + let ret = { 540 540 + match Ordering::TYPE { 541 541 + OrderingType::Full => T::Repr::atomic_fetch_add(&self.0, v), 542 542 + OrderingType::Acquire => T::Repr::atomic_fetch_add_acquire(&self.0, v), 543 543 + OrderingType::Release => T::Repr::atomic_fetch_add_release(&self.0, v), 544 544 + OrderingType::Relaxed => T::Repr::atomic_fetch_add_relaxed(&self.0, v), 545 545 + } 546 546 + }; 547 547 + 548 548 + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. 549 549 + unsafe { from_repr(ret) } 550 550 + } 551 551 + }

+265

rust/kernel/sync/atomic/internal.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Atomic internal implementations. 4 4 + //! 5 5 + //! Provides 1:1 mapping to the C atomic operations. 6 6 + 7 7 + use crate::bindings; 8 8 + use crate::macros::paste; 9 9 + use core::cell::UnsafeCell; 10 10 + 11 11 + mod private { 12 12 + /// Sealed trait marker to disable customized impls on atomic implementation traits. 13 13 + pub trait Sealed {} 14 14 + } 15 15 + 16 16 + // `i32` and `i64` are only supported atomic implementations. 17 17 + impl private::Sealed for i32 {} 18 18 + impl private::Sealed for i64 {} 19 19 + 20 20 + /// A marker trait for types that implement atomic operations with C side primitives. 21 21 + /// 22 22 + /// This trait is sealed, and only types that have directly mapping to the C side atomics should 23 23 + /// impl this: 24 24 + /// 25 25 + /// - `i32` maps to `atomic_t`. 26 26 + /// - `i64` maps to `atomic64_t`. 27 27 + pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { 28 28 + /// The type of the delta in arithmetic or logical operations. 29 29 + /// 30 30 + /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of 31 31 + /// [`Self`], but it may be different for the atomic pointer type. 32 32 + type Delta; 33 33 + } 34 34 + 35 35 + // `atomic_t` implements atomic operations on `i32`. 36 36 + impl AtomicImpl for i32 { 37 37 + type Delta = Self; 38 38 + } 39 39 + 40 40 + // `atomic64_t` implements atomic operations on `i64`. 41 41 + impl AtomicImpl for i64 { 42 42 + type Delta = Self; 43 43 + } 44 44 + 45 45 + /// Atomic representation. 46 46 + #[repr(transparent)] 47 47 + pub struct AtomicRepr<T: AtomicImpl>(UnsafeCell<T>); 48 48 + 49 49 + impl<T: AtomicImpl> AtomicRepr<T> { 50 50 + /// Creates a new atomic representation `T`. 51 51 + pub const fn new(v: T) -> Self { 52 52 + Self(UnsafeCell::new(v)) 53 53 + } 54 54 + 55 55 + /// Returns a pointer to the underlying `T`. 56 56 + /// 57 57 + /// # Guarantees 58 58 + /// 59 59 + /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]). 60 60 + pub const fn as_ptr(&self) -> *mut T { 61 61 + // GUARANTEE: `self.0` is an `UnsafeCell<T>`, therefore the pointer returned by `.get()` 62 62 + // must be valid and properly aligned. 63 63 + self.0.get() 64 64 + } 65 65 + } 66 66 + 67 67 + // This macro generates the function signature with given argument list and return type. 68 68 + macro_rules! declare_atomic_method { 69 69 + ( 70 70 + $(#[doc=$doc:expr])* 71 71 + $func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)? 72 72 + ) => { 73 73 + paste!( 74 74 + $(#[doc = $doc])* 75 75 + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?; 76 76 + ); 77 77 + }; 78 78 + ( 79 79 + $(#[doc=$doc:expr])* 80 80 + $func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? 81 81 + ) => { 82 82 + paste!( 83 83 + declare_atomic_method!( 84 84 + $(#[doc = $doc])* 85 85 + [< $func _ $variant >]($($arg_sig)*) $(-> $ret)? 86 86 + ); 87 87 + ); 88 88 + 89 89 + declare_atomic_method!( 90 90 + $(#[doc = $doc])* 91 91 + $func [$($rest)*]($($arg_sig)*) $(-> $ret)? 92 92 + ); 93 93 + }; 94 94 + ( 95 95 + $(#[doc=$doc:expr])* 96 96 + $func:ident []($($arg_sig:tt)*) $(-> $ret:ty)? 97 97 + ) => { 98 98 + declare_atomic_method!( 99 99 + $(#[doc = $doc])* 100 100 + $func($($arg_sig)*) $(-> $ret)? 101 101 + ); 102 102 + } 103 103 + } 104 104 + 105 105 + // This macro generates the function implementation with given argument list and return type, and it 106 106 + // will replace "call(...)" expression with "$ctype _ $func" to call the real C function. 107 107 + macro_rules! impl_atomic_method { 108 108 + ( 109 109 + ($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? { 110 110 + $unsafe:tt { call($($c_arg:expr),*) } 111 111 + } 112 112 + ) => { 113 113 + paste!( 114 114 + #[inline(always)] 115 115 + fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? { 116 116 + // TODO: Ideally we want to use the SAFETY comments written at the macro invocation 117 117 + // (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments 118 118 + // are just comments, and they are not passed to macros as tokens, therefore we 119 119 + // cannot use them here. One potential improvement is that if we support using 120 120 + // attributes as an alternative for SAFETY comments, then we can use that for macro 121 121 + // generating code. 122 122 + // 123 123 + // SAFETY: specified on macro invocation. 124 124 + $unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) } 125 125 + } 126 126 + ); 127 127 + }; 128 128 + ( 129 129 + ($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? { 130 130 + $unsafe:tt { call($($arg:tt)*) } 131 131 + } 132 132 + ) => { 133 133 + paste!( 134 134 + impl_atomic_method!( 135 135 + ($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? { 136 136 + $unsafe { call($($arg)*) } 137 137 + } 138 138 + ); 139 139 + ); 140 140 + impl_atomic_method!( 141 141 + ($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? { 142 142 + $unsafe { call($($arg)*) } 143 143 + } 144 144 + ); 145 145 + }; 146 146 + ( 147 147 + ($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? { 148 148 + $unsafe:tt { call($($arg:tt)*) } 149 149 + } 150 150 + ) => { 151 151 + impl_atomic_method!( 152 152 + ($ctype) $func($($arg_sig)*) $(-> $ret)? { 153 153 + $unsafe { call($($arg)*) } 154 154 + } 155 155 + ); 156 156 + } 157 157 + } 158 158 + 159 159 + // Delcares $ops trait with methods and implements the trait for `i32` and `i64`. 160 160 + macro_rules! declare_and_impl_atomic_methods { 161 161 + ($(#[$attr:meta])* $pub:vis trait $ops:ident { 162 162 + $( 163 163 + $(#[doc=$doc:expr])* 164 164 + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { 165 165 + $unsafe:tt { bindings::#call($($arg:tt)*) } 166 166 + } 167 167 + )* 168 168 + }) => { 169 169 + $(#[$attr])* 170 170 + $pub trait $ops: AtomicImpl { 171 171 + $( 172 172 + declare_atomic_method!( 173 173 + $(#[doc=$doc])* 174 174 + $func[$($variant)*]($($arg_sig)*) $(-> $ret)? 175 175 + ); 176 176 + )* 177 177 + } 178 178 + 179 179 + impl $ops for i32 { 180 180 + $( 181 181 + impl_atomic_method!( 182 182 + (atomic) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { 183 183 + $unsafe { call($($arg)*) } 184 184 + } 185 185 + ); 186 186 + )* 187 187 + } 188 188 + 189 189 + impl $ops for i64 { 190 190 + $( 191 191 + impl_atomic_method!( 192 192 + (atomic64) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { 193 193 + $unsafe { call($($arg)*) } 194 194 + } 195 195 + ); 196 196 + )* 197 197 + } 198 198 + } 199 199 + } 200 200 + 201 201 + declare_and_impl_atomic_methods!( 202 202 + /// Basic atomic operations 203 203 + pub trait AtomicBasicOps { 204 204 + /// Atomic read (load). 205 205 + fn read[acquire](a: &AtomicRepr<Self>) -> Self { 206 206 + // SAFETY: `a.as_ptr()` is valid and properly aligned. 207 207 + unsafe { bindings::#call(a.as_ptr().cast()) } 208 208 + } 209 209 + 210 210 + /// Atomic set (store). 211 211 + fn set[release](a: &AtomicRepr<Self>, v: Self) { 212 212 + // SAFETY: `a.as_ptr()` is valid and properly aligned. 213 213 + unsafe { bindings::#call(a.as_ptr().cast(), v) } 214 214 + } 215 215 + } 216 216 + ); 217 217 + 218 218 + declare_and_impl_atomic_methods!( 219 219 + /// Exchange and compare-and-exchange atomic operations 220 220 + pub trait AtomicExchangeOps { 221 221 + /// Atomic exchange. 222 222 + /// 223 223 + /// Atomically updates `*a` to `v` and returns the old value. 224 224 + fn xchg[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self) -> Self { 225 225 + // SAFETY: `a.as_ptr()` is valid and properly aligned. 226 226 + unsafe { bindings::#call(a.as_ptr().cast(), v) } 227 227 + } 228 228 + 229 229 + /// Atomic compare and exchange. 230 230 + /// 231 231 + /// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not 232 232 + /// modified, `*old` is updated to the current value of `*a`. 233 233 + /// 234 234 + /// Return `true` if the update of `*a` occurred, `false` otherwise. 235 235 + fn try_cmpxchg[acquire, release, relaxed]( 236 236 + a: &AtomicRepr<Self>, old: &mut Self, new: Self 237 237 + ) -> bool { 238 238 + // SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)` 239 239 + // is valid and properly aligned. 240 240 + unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) } 241 241 + } 242 242 + } 243 243 + ); 244 244 + 245 245 + declare_and_impl_atomic_methods!( 246 246 + /// Atomic arithmetic operations 247 247 + pub trait AtomicArithmeticOps { 248 248 + /// Atomic add (wrapping). 249 249 + /// 250 250 + /// Atomically updates `*a` to `(*a).wrapping_add(v)`. 251 251 + fn add[](a: &AtomicRepr<Self>, v: Self::Delta) { 252 252 + // SAFETY: `a.as_ptr()` is valid and properly aligned. 253 253 + unsafe { bindings::#call(v, a.as_ptr().cast()) } 254 254 + } 255 255 + 256 256 + /// Atomic fetch and add (wrapping). 257 257 + /// 258 258 + /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` 259 259 + /// before the update. 260 260 + fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { 261 261 + // SAFETY: `a.as_ptr()` is valid and properly aligned. 262 262 + unsafe { bindings::#call(v, a.as_ptr().cast()) } 263 263 + } 264 264 + } 265 265 + );

+104

rust/kernel/sync/atomic/ordering.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Memory orderings. 4 4 + //! 5 5 + //! The semantics of these orderings follows the [`LKMM`] definitions and rules. 6 6 + //! 7 7 + //! - [`Acquire`] provides ordering between the load part of the annotated operation and all the 8 8 + //! following memory accesses, and if there is a store part, the store part has the [`Relaxed`] 9 9 + //! ordering. 10 10 + //! - [`Release`] provides ordering between all the preceding memory accesses and the store part of 11 11 + //! the annotated operation, and if there is a load part, the load part has the [`Relaxed`] 12 12 + //! ordering. 13 13 + //! - [`Full`] means "fully-ordered", that is: 14 14 + //! - It provides ordering between all the preceding memory accesses and the annotated operation. 15 15 + //! - It provides ordering between the annotated operation and all the following memory accesses. 16 16 + //! - It provides ordering between all the preceding memory accesses and all the following memory 17 17 + //! accesses. 18 18 + //! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). 19 19 + //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are 20 20 + //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. 21 21 + //! 22 22 + //! [`LKMM`]: srctree/tools/memory-model/ 23 23 + //! [`explanation`]: srctree/tools/memory-model/Documentation/explanation.txt 24 24 + 25 25 + /// The annotation type for relaxed memory ordering, for the description of relaxed memory 26 26 + /// ordering, see [module-level documentation]. 27 27 + /// 28 28 + /// [module-level documentation]: crate::sync::atomic::ordering 29 29 + pub struct Relaxed; 30 30 + 31 31 + /// The annotation type for acquire memory ordering, for the description of acquire memory 32 32 + /// ordering, see [module-level documentation]. 33 33 + /// 34 34 + /// [module-level documentation]: crate::sync::atomic::ordering 35 35 + pub struct Acquire; 36 36 + 37 37 + /// The annotation type for release memory ordering, for the description of release memory 38 38 + /// ordering, see [module-level documentation]. 39 39 + /// 40 40 + /// [module-level documentation]: crate::sync::atomic::ordering 41 41 + pub struct Release; 42 42 + 43 43 + /// The annotation type for fully-ordered memory ordering, for the description fully-ordered memory 44 44 + /// ordering, see [module-level documentation]. 45 45 + /// 46 46 + /// [module-level documentation]: crate::sync::atomic::ordering 47 47 + pub struct Full; 48 48 + 49 49 + /// Describes the exact memory ordering. 50 50 + #[doc(hidden)] 51 51 + pub enum OrderingType { 52 52 + /// Relaxed ordering. 53 53 + Relaxed, 54 54 + /// Acquire ordering. 55 55 + Acquire, 56 56 + /// Release ordering. 57 57 + Release, 58 58 + /// Fully-ordered. 59 59 + Full, 60 60 + } 61 61 + 62 62 + mod internal { 63 63 + /// Sealed trait, can be only implemented inside atomic mod. 64 64 + pub trait Sealed {} 65 65 + 66 66 + impl Sealed for super::Relaxed {} 67 67 + impl Sealed for super::Acquire {} 68 68 + impl Sealed for super::Release {} 69 69 + impl Sealed for super::Full {} 70 70 + } 71 71 + 72 72 + /// The trait bound for annotating operations that support any ordering. 73 73 + pub trait Ordering: internal::Sealed { 74 74 + /// Describes the exact memory ordering. 75 75 + const TYPE: OrderingType; 76 76 + } 77 77 + 78 78 + impl Ordering for Relaxed { 79 79 + const TYPE: OrderingType = OrderingType::Relaxed; 80 80 + } 81 81 + 82 82 + impl Ordering for Acquire { 83 83 + const TYPE: OrderingType = OrderingType::Acquire; 84 84 + } 85 85 + 86 86 + impl Ordering for Release { 87 87 + const TYPE: OrderingType = OrderingType::Release; 88 88 + } 89 89 + 90 90 + impl Ordering for Full { 91 91 + const TYPE: OrderingType = OrderingType::Full; 92 92 + } 93 93 + 94 94 + /// The trait bound for operations that only support acquire or relaxed ordering. 95 95 + pub trait AcquireOrRelaxed: Ordering {} 96 96 + 97 97 + impl AcquireOrRelaxed for Acquire {} 98 98 + impl AcquireOrRelaxed for Relaxed {} 99 99 + 100 100 + /// The trait bound for operations that only support release or relaxed ordering. 101 101 + pub trait ReleaseOrRelaxed: Ordering {} 102 102 + 103 103 + impl ReleaseOrRelaxed for Release {} 104 104 + impl ReleaseOrRelaxed for Relaxed {}

+169

rust/kernel/sync/atomic/predefine.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Pre-defined atomic types 4 4 + 5 5 + use crate::static_assert; 6 6 + use core::mem::{align_of, size_of}; 7 7 + 8 8 + // SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to 9 9 + // itself. 10 10 + unsafe impl super::AtomicType for i32 { 11 11 + type Repr = i32; 12 12 + } 13 13 + 14 14 + // SAFETY: The wrapping add result of two `i32`s is a valid `i32`. 15 15 + unsafe impl super::AtomicAdd<i32> for i32 { 16 16 + fn rhs_into_delta(rhs: i32) -> i32 { 17 17 + rhs 18 18 + } 19 19 + } 20 20 + 21 21 + // SAFETY: `i64` has the same size and alignment with itself, and is round-trip transmutable to 22 22 + // itself. 23 23 + unsafe impl super::AtomicType for i64 { 24 24 + type Repr = i64; 25 25 + } 26 26 + 27 27 + // SAFETY: The wrapping add result of two `i64`s is a valid `i64`. 28 28 + unsafe impl super::AtomicAdd<i64> for i64 { 29 29 + fn rhs_into_delta(rhs: i64) -> i64 { 30 30 + rhs 31 31 + } 32 32 + } 33 33 + 34 34 + // Defines an internal type that always maps to the integer type which has the same size alignment 35 35 + // as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to 36 36 + // `isize_atomic_repr`, which also always implements `AtomicImpl`. 37 37 + #[allow(non_camel_case_types)] 38 38 + #[cfg(not(CONFIG_64BIT))] 39 39 + type isize_atomic_repr = i32; 40 40 + #[allow(non_camel_case_types)] 41 41 + #[cfg(CONFIG_64BIT)] 42 42 + type isize_atomic_repr = i64; 43 43 + 44 44 + // Ensure size and alignment requirements are checked. 45 45 + static_assert!(size_of::<isize>() == size_of::<isize_atomic_repr>()); 46 46 + static_assert!(align_of::<isize>() == align_of::<isize_atomic_repr>()); 47 47 + static_assert!(size_of::<usize>() == size_of::<isize_atomic_repr>()); 48 48 + static_assert!(align_of::<usize>() == align_of::<isize_atomic_repr>()); 49 49 + 50 50 + // SAFETY: `isize` has the same size and alignment with `isize_atomic_repr`, and is round-trip 51 51 + // transmutable to `isize_atomic_repr`. 52 52 + unsafe impl super::AtomicType for isize { 53 53 + type Repr = isize_atomic_repr; 54 54 + } 55 55 + 56 56 + // SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. 57 57 + unsafe impl super::AtomicAdd<isize> for isize { 58 58 + fn rhs_into_delta(rhs: isize) -> isize_atomic_repr { 59 59 + rhs as isize_atomic_repr 60 60 + } 61 61 + } 62 62 + 63 63 + // SAFETY: `u32` and `i32` has the same size and alignment, and `u32` is round-trip transmutable to 64 64 + // `i32`. 65 65 + unsafe impl super::AtomicType for u32 { 66 66 + type Repr = i32; 67 67 + } 68 68 + 69 69 + // SAFETY: The wrapping add result of two `i32`s is a valid `u32`. 70 70 + unsafe impl super::AtomicAdd<u32> for u32 { 71 71 + fn rhs_into_delta(rhs: u32) -> i32 { 72 72 + rhs as i32 73 73 + } 74 74 + } 75 75 + 76 76 + // SAFETY: `u64` and `i64` has the same size and alignment, and `u64` is round-trip transmutable to 77 77 + // `i64`. 78 78 + unsafe impl super::AtomicType for u64 { 79 79 + type Repr = i64; 80 80 + } 81 81 + 82 82 + // SAFETY: The wrapping add result of two `i64`s is a valid `u64`. 83 83 + unsafe impl super::AtomicAdd<u64> for u64 { 84 84 + fn rhs_into_delta(rhs: u64) -> i64 { 85 85 + rhs as i64 86 86 + } 87 87 + } 88 88 + 89 89 + // SAFETY: `usize` has the same size and alignment with `isize_atomic_repr`, and is round-trip 90 90 + // transmutable to `isize_atomic_repr`. 91 91 + unsafe impl super::AtomicType for usize { 92 92 + type Repr = isize_atomic_repr; 93 93 + } 94 94 + 95 95 + // SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. 96 96 + unsafe impl super::AtomicAdd<usize> for usize { 97 97 + fn rhs_into_delta(rhs: usize) -> isize_atomic_repr { 98 98 + rhs as isize_atomic_repr 99 99 + } 100 100 + } 101 101 + 102 102 + use crate::macros::kunit_tests; 103 103 + 104 104 + #[kunit_tests(rust_atomics)] 105 105 + mod tests { 106 106 + use super::super::*; 107 107 + 108 108 + // Call $fn($val) with each $type of $val. 109 109 + macro_rules! for_each_type { 110 110 + ($val:literal in [$($type:ty),*] $fn:expr) => { 111 111 + $({ 112 112 + let v: $type = $val; 113 113 + 114 114 + $fn(v); 115 115 + })* 116 116 + } 117 117 + } 118 118 + 119 119 + #[test] 120 120 + fn atomic_basic_tests() { 121 121 + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { 122 122 + let x = Atomic::new(v); 123 123 + 124 124 + assert_eq!(v, x.load(Relaxed)); 125 125 + }); 126 126 + } 127 127 + 128 128 + #[test] 129 129 + fn atomic_xchg_tests() { 130 130 + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { 131 131 + let x = Atomic::new(v); 132 132 + 133 133 + let old = v; 134 134 + let new = v + 1; 135 135 + 136 136 + assert_eq!(old, x.xchg(new, Full)); 137 137 + assert_eq!(new, x.load(Relaxed)); 138 138 + }); 139 139 + } 140 140 + 141 141 + #[test] 142 142 + fn atomic_cmpxchg_tests() { 143 143 + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { 144 144 + let x = Atomic::new(v); 145 145 + 146 146 + let old = v; 147 147 + let new = v + 1; 148 148 + 149 149 + assert_eq!(Err(old), x.cmpxchg(new, new, Full)); 150 150 + assert_eq!(old, x.load(Relaxed)); 151 151 + assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); 152 152 + assert_eq!(new, x.load(Relaxed)); 153 153 + }); 154 154 + } 155 155 + 156 156 + #[test] 157 157 + fn atomic_arithmetic_tests() { 158 158 + for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { 159 159 + let x = Atomic::new(v); 160 160 + 161 161 + assert_eq!(v, x.fetch_add(12, Full)); 162 162 + assert_eq!(v + 12, x.load(Relaxed)); 163 163 + 164 164 + x.add(13, Relaxed); 165 165 + 166 166 + assert_eq!(v + 25, x.load(Relaxed)); 167 167 + }); 168 168 + } 169 169 + }

+61

rust/kernel/sync/barrier.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Memory barriers. 4 4 + //! 5 5 + //! These primitives have the same semantics as their C counterparts: and the precise definitions 6 6 + //! of semantics can be found at [`LKMM`]. 7 7 + //! 8 8 + //! [`LKMM`]: srctree/tools/memory-model/ 9 9 + 10 10 + /// A compiler barrier. 11 11 + /// 12 12 + /// A barrier that prevents compiler from reordering memory accesses across the barrier. 13 13 + #[inline(always)] 14 14 + pub(crate) fn barrier() { 15 15 + // By default, Rust inline asms are treated as being able to access any memory or flags, hence 16 16 + // it suffices as a compiler barrier. 17 17 + // 18 18 + // SAFETY: An empty asm block. 19 19 + unsafe { core::arch::asm!("") }; 20 20 + } 21 21 + 22 22 + /// A full memory barrier. 23 23 + /// 24 24 + /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. 25 25 + #[inline(always)] 26 26 + pub fn smp_mb() { 27 27 + if cfg!(CONFIG_SMP) { 28 28 + // SAFETY: `smp_mb()` is safe to call. 29 29 + unsafe { bindings::smp_mb() }; 30 30 + } else { 31 31 + barrier(); 32 32 + } 33 33 + } 34 34 + 35 35 + /// A write-write memory barrier. 36 36 + /// 37 37 + /// A barrier that prevents compiler and CPU from reordering memory write accesses across the 38 38 + /// barrier. 39 39 + #[inline(always)] 40 40 + pub fn smp_wmb() { 41 41 + if cfg!(CONFIG_SMP) { 42 42 + // SAFETY: `smp_wmb()` is safe to call. 43 43 + unsafe { bindings::smp_wmb() }; 44 44 + } else { 45 45 + barrier(); 46 46 + } 47 47 + } 48 48 + 49 49 + /// A read-read memory barrier. 50 50 + /// 51 51 + /// A barrier that prevents compiler and CPU from reordering memory read accesses across the 52 52 + /// barrier. 53 53 + #[inline(always)] 54 54 + pub fn smp_rmb() { 55 55 + if cfg!(CONFIG_SMP) { 56 56 + // SAFETY: `smp_rmb()` is safe to call. 57 57 + unsafe { bindings::smp_rmb() }; 58 58 + } else { 59 59 + barrier(); 60 60 + } 61 61 + }

+113

rust/kernel/sync/refcount.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Atomic reference counting. 4 4 + //! 5 5 + //! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h) 6 6 + 7 7 + use crate::build_assert; 8 8 + use crate::sync::atomic::Atomic; 9 9 + use crate::types::Opaque; 10 10 + 11 11 + /// Atomic reference counter. 12 12 + /// 13 13 + /// This type is conceptually an atomic integer, but provides saturation semantics compared to 14 14 + /// normal atomic integers. Values in the negative range when viewed as a signed integer are 15 15 + /// saturation (bad) values. For details about the saturation semantics, please refer to top of 16 16 + /// [`include/linux/refcount.h`](srctree/include/linux/refcount.h). 17 17 + /// 18 18 + /// Wraps the kernel's C `refcount_t`. 19 19 + #[repr(transparent)] 20 20 + pub struct Refcount(Opaque<bindings::refcount_t>); 21 21 + 22 22 + impl Refcount { 23 23 + /// Construct a new [`Refcount`] from an initial value. 24 24 + /// 25 25 + /// The initial value should be non-saturated. 26 26 + #[inline] 27 27 + pub fn new(value: i32) -> Self { 28 28 + build_assert!(value >= 0, "initial value saturated"); 29 29 + // SAFETY: There are no safety requirements for this FFI call. 30 30 + Self(Opaque::new(unsafe { bindings::REFCOUNT_INIT(value) })) 31 31 + } 32 32 + 33 33 + #[inline] 34 34 + fn as_ptr(&self) -> *mut bindings::refcount_t { 35 35 + self.0.get() 36 36 + } 37 37 + 38 38 + /// Get the underlying atomic counter that backs the refcount. 39 39 + /// 40 40 + /// NOTE: Usage of this function is discouraged as it can circumvent the protections offered by 41 41 + /// `refcount.h`. If there is no way to achieve the result using APIs in `refcount.h`, then 42 42 + /// this function can be used. Otherwise consider adding a binding for the required API. 43 43 + #[inline] 44 44 + pub fn as_atomic(&self) -> &Atomic<i32> { 45 45 + let ptr = self.0.get().cast(); 46 46 + // SAFETY: `refcount_t` is a transparent wrapper of `atomic_t`, which is an atomic 32-bit 47 47 + // integer that is layout-wise compatible with `Atomic<i32>`. All values are valid for 48 48 + // `refcount_t`, despite some of the values being considered saturated and "bad". 49 49 + unsafe { &*ptr } 50 50 + } 51 51 + 52 52 + /// Set a refcount's value. 53 53 + #[inline] 54 54 + pub fn set(&self, value: i32) { 55 55 + // SAFETY: `self.as_ptr()` is valid. 56 56 + unsafe { bindings::refcount_set(self.as_ptr(), value) } 57 57 + } 58 58 + 59 59 + /// Increment a refcount. 60 60 + /// 61 61 + /// It will saturate if overflows and `WARN`. It will also `WARN` if the refcount is 0, as this 62 62 + /// represents a possible use-after-free condition. 63 63 + /// 64 64 + /// Provides no memory ordering, it is assumed that caller already has a reference on the 65 65 + /// object. 66 66 + #[inline] 67 67 + pub fn inc(&self) { 68 68 + // SAFETY: self is valid. 69 69 + unsafe { bindings::refcount_inc(self.as_ptr()) } 70 70 + } 71 71 + 72 72 + /// Decrement a refcount. 73 73 + /// 74 74 + /// It will `WARN` on underflow and fail to decrement when saturated. 75 75 + /// 76 76 + /// Provides release memory ordering, such that prior loads and stores are done 77 77 + /// before. 78 78 + #[inline] 79 79 + pub fn dec(&self) { 80 80 + // SAFETY: `self.as_ptr()` is valid. 81 81 + unsafe { bindings::refcount_dec(self.as_ptr()) } 82 82 + } 83 83 + 84 84 + /// Decrement a refcount and test if it is 0. 85 85 + /// 86 86 + /// It will `WARN` on underflow and fail to decrement when saturated. 87 87 + /// 88 88 + /// Provides release memory ordering, such that prior loads and stores are done 89 89 + /// before, and provides an acquire ordering on success such that memory deallocation 90 90 + /// must come after. 91 91 + /// 92 92 + /// Returns true if the resulting refcount is 0, false otherwise. 93 93 + /// 94 94 + /// # Notes 95 95 + /// 96 96 + /// A common pattern of using `Refcount` is to free memory when the reference count reaches 97 97 + /// zero. This means that the reference to `Refcount` could become invalid after calling this 98 98 + /// function. This is fine as long as the reference to `Refcount` is no longer used when this 99 99 + /// function returns `false`. It is not necessary to use raw pointers in this scenario, see 100 100 + /// <https://github.com/rust-lang/rust/issues/55005>. 101 101 + #[inline] 102 102 + #[must_use = "use `dec` instead if you do not need to test if it is 0"] 103 103 + pub fn dec_and_test(&self) -> bool { 104 104 + // SAFETY: `self.as_ptr()` is valid. 105 105 + unsafe { bindings::refcount_dec_and_test(self.as_ptr()) } 106 106 + } 107 107 + } 108 108 + 109 109 + // SAFETY: `refcount_t` is thread-safe. 110 110 + unsafe impl Send for Refcount {} 111 111 + 112 112 + // SAFETY: `refcount_t` is thread-safe. 113 113 + unsafe impl Sync for Refcount {}

+1

scripts/atomic/gen-atomics.sh

reviewed

··· 11 11 gen-atomic-instrumented.sh linux/atomic/atomic-instrumented.h 12 12 gen-atomic-long.sh linux/atomic/atomic-long.h 13 13 gen-atomic-fallback.sh linux/atomic/atomic-arch-fallback.h 14 14 + gen-rust-atomic-helpers.sh ../rust/helpers/atomic.c 14 15 EOF 15 16 while read script header args; do 16 17 /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}

+67

scripts/atomic/gen-rust-atomic-helpers.sh

reviewed

··· 1 1 + #!/bin/sh 2 2 + # SPDX-License-Identifier: GPL-2.0 3 3 + 4 4 + ATOMICDIR=$(dirname $0) 5 5 + 6 6 + . ${ATOMICDIR}/atomic-tbl.sh 7 7 + 8 8 + #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) 9 9 + gen_proto_order_variant() 10 10 + { 11 11 + local meta="$1"; shift 12 12 + local pfx="$1"; shift 13 13 + local name="$1"; shift 14 14 + local sfx="$1"; shift 15 15 + local order="$1"; shift 16 16 + local atomic="$1"; shift 17 17 + local int="$1"; shift 18 18 + 19 19 + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" 20 20 + 21 21 + local ret="$(gen_ret_type "${meta}" "${int}")" 22 22 + local params="$(gen_params "${int}" "${atomic}" "$@")" 23 23 + local args="$(gen_args "$@")" 24 24 + local retstmt="$(gen_ret_stmt "${meta}")" 25 25 + 26 26 + cat <<EOF 27 27 + __rust_helper ${ret} 28 28 + rust_helper_${atomicname}(${params}) 29 29 + { 30 30 + ${retstmt}${atomicname}(${args}); 31 31 + } 32 32 + 33 33 + EOF 34 34 + } 35 35 + 36 36 + cat << EOF 37 37 + // SPDX-License-Identifier: GPL-2.0 38 38 + 39 39 + // Generated by $0 40 40 + // DO NOT MODIFY THIS FILE DIRECTLY 41 41 + 42 42 + /* 43 43 + * This file provides helpers for the various atomic functions for Rust. 44 44 + */ 45 45 + #ifndef _RUST_ATOMIC_API_H 46 46 + #define _RUST_ATOMIC_API_H 47 47 + 48 48 + #include <linux/atomic.h> 49 49 + 50 50 + // TODO: Remove this after INLINE_HELPERS support is added. 51 51 + #ifndef __rust_helper 52 52 + #define __rust_helper 53 53 + #endif 54 54 + 55 55 + EOF 56 56 + 57 57 + grep '^[a-z]' "$1" | while read name meta args; do 58 58 + gen_proto "${meta}" "${name}" "atomic" "int" ${args} 59 59 + done 60 60 + 61 61 + grep '^[a-z]' "$1" | while read name meta args; do 62 62 + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} 63 63 + done 64 64 + 65 65 + cat <<EOF 66 66 + #endif /* _RUST_ATOMIC_API_H */ 67 67 + EOF