Merge tag 'rcu_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+235 -1

include/linux/atomic-fallback.h

··· 77 77 78 78 #endif /* cmpxchg64_relaxed */ 79 79 80 + #define arch_atomic_read atomic_read 81 + #define arch_atomic_read_acquire atomic_read_acquire 82 + 80 83 #ifndef atomic_read_acquire 81 84 static __always_inline int 82 85 atomic_read_acquire(const atomic_t *v) ··· 89 86 #define atomic_read_acquire atomic_read_acquire 90 87 #endif 91 88 89 + #define arch_atomic_set atomic_set 90 + #define arch_atomic_set_release atomic_set_release 91 + 92 92 #ifndef atomic_set_release 93 93 static __always_inline void 94 94 atomic_set_release(atomic_t *v, int i) ··· 100 94 } 101 95 #define atomic_set_release atomic_set_release 102 96 #endif 97 + 98 + #define arch_atomic_add atomic_add 99 + 100 + #define arch_atomic_add_return atomic_add_return 101 + #define arch_atomic_add_return_acquire atomic_add_return_acquire 102 + #define arch_atomic_add_return_release atomic_add_return_release 103 + #define arch_atomic_add_return_relaxed atomic_add_return_relaxed 103 104 104 105 #ifndef atomic_add_return_relaxed 105 106 #define atomic_add_return_acquire atomic_add_return ··· 150 137 151 138 #endif /* atomic_add_return_relaxed */ 152 139 140 + #define arch_atomic_fetch_add atomic_fetch_add 141 + #define arch_atomic_fetch_add_acquire atomic_fetch_add_acquire 142 + #define arch_atomic_fetch_add_release atomic_fetch_add_release 143 + #define arch_atomic_fetch_add_relaxed atomic_fetch_add_relaxed 144 + 153 145 #ifndef atomic_fetch_add_relaxed 154 146 #define atomic_fetch_add_acquire atomic_fetch_add 155 147 #define atomic_fetch_add_release atomic_fetch_add ··· 196 178 #endif 197 179 198 180 #endif /* atomic_fetch_add_relaxed */ 181 + 182 + #define arch_atomic_sub atomic_sub 183 + 184 + #define arch_atomic_sub_return atomic_sub_return 185 + #define arch_atomic_sub_return_acquire atomic_sub_return_acquire 186 + #define arch_atomic_sub_return_release atomic_sub_return_release 187 + #define arch_atomic_sub_return_relaxed atomic_sub_return_relaxed 199 188 200 189 #ifndef atomic_sub_return_relaxed 201 190 #define atomic_sub_return_acquire atomic_sub_return ··· 246 221 247 222 #endif /* atomic_sub_return_relaxed */ 248 223 224 + #define arch_atomic_fetch_sub atomic_fetch_sub 225 + #define arch_atomic_fetch_sub_acquire atomic_fetch_sub_acquire 226 + #define arch_atomic_fetch_sub_release atomic_fetch_sub_release 227 + #define arch_atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 228 + 249 229 #ifndef atomic_fetch_sub_relaxed 250 230 #define atomic_fetch_sub_acquire atomic_fetch_sub 251 231 #define atomic_fetch_sub_release atomic_fetch_sub ··· 293 263 294 264 #endif /* atomic_fetch_sub_relaxed */ 295 265 266 + #define arch_atomic_inc atomic_inc 267 + 296 268 #ifndef atomic_inc 297 269 static __always_inline void 298 270 atomic_inc(atomic_t *v) ··· 303 271 } 304 272 #define atomic_inc atomic_inc 305 273 #endif 274 + 275 + #define arch_atomic_inc_return atomic_inc_return 276 + #define arch_atomic_inc_return_acquire atomic_inc_return_acquire 277 + #define arch_atomic_inc_return_release atomic_inc_return_release 278 + #define arch_atomic_inc_return_relaxed atomic_inc_return_relaxed 306 279 307 280 #ifndef atomic_inc_return_relaxed 308 281 #ifdef atomic_inc_return ··· 390 353 391 354 #endif /* atomic_inc_return_relaxed */ 392 355 356 + #define arch_atomic_fetch_inc atomic_fetch_inc 357 + #define arch_atomic_fetch_inc_acquire atomic_fetch_inc_acquire 358 + #define arch_atomic_fetch_inc_release atomic_fetch_inc_release 359 + #define arch_atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed 360 + 393 361 #ifndef atomic_fetch_inc_relaxed 394 362 #ifdef atomic_fetch_inc 395 363 #define atomic_fetch_inc_acquire atomic_fetch_inc ··· 476 434 477 435 #endif /* atomic_fetch_inc_relaxed */ 478 436 437 + #define arch_atomic_dec atomic_dec 438 + 479 439 #ifndef atomic_dec 480 440 static __always_inline void 481 441 atomic_dec(atomic_t *v) ··· 486 442 } 487 443 #define atomic_dec atomic_dec 488 444 #endif 445 + 446 + #define arch_atomic_dec_return atomic_dec_return 447 + #define arch_atomic_dec_return_acquire atomic_dec_return_acquire 448 + #define arch_atomic_dec_return_release atomic_dec_return_release 449 + #define arch_atomic_dec_return_relaxed atomic_dec_return_relaxed 489 450 490 451 #ifndef atomic_dec_return_relaxed 491 452 #ifdef atomic_dec_return ··· 573 524 574 525 #endif /* atomic_dec_return_relaxed */ 575 526 527 + #define arch_atomic_fetch_dec atomic_fetch_dec 528 + #define arch_atomic_fetch_dec_acquire atomic_fetch_dec_acquire 529 + #define arch_atomic_fetch_dec_release atomic_fetch_dec_release 530 + #define arch_atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed 531 + 576 532 #ifndef atomic_fetch_dec_relaxed 577 533 #ifdef atomic_fetch_dec 578 534 #define atomic_fetch_dec_acquire atomic_fetch_dec ··· 659 605 660 606 #endif /* atomic_fetch_dec_relaxed */ 661 607 608 + #define arch_atomic_and atomic_and 609 + 610 + #define arch_atomic_fetch_and atomic_fetch_and 611 + #define arch_atomic_fetch_and_acquire atomic_fetch_and_acquire 612 + #define arch_atomic_fetch_and_release atomic_fetch_and_release 613 + #define arch_atomic_fetch_and_relaxed atomic_fetch_and_relaxed 614 + 662 615 #ifndef atomic_fetch_and_relaxed 663 616 #define atomic_fetch_and_acquire atomic_fetch_and 664 617 #define atomic_fetch_and_release atomic_fetch_and ··· 708 647 709 648 #endif /* atomic_fetch_and_relaxed */ 710 649 650 + #define arch_atomic_andnot atomic_andnot 651 + 711 652 #ifndef atomic_andnot 712 653 static __always_inline void 713 654 atomic_andnot(int i, atomic_t *v) ··· 718 655 } 719 656 #define atomic_andnot atomic_andnot 720 657 #endif 658 + 659 + #define arch_atomic_fetch_andnot atomic_fetch_andnot 660 + #define arch_atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire 661 + #define arch_atomic_fetch_andnot_release atomic_fetch_andnot_release 662 + #define arch_atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed 721 663 722 664 #ifndef atomic_fetch_andnot_relaxed 723 665 #ifdef atomic_fetch_andnot ··· 805 737 806 738 #endif /* atomic_fetch_andnot_relaxed */ 807 739 740 + #define arch_atomic_or atomic_or 741 + 742 + #define arch_atomic_fetch_or atomic_fetch_or 743 + #define arch_atomic_fetch_or_acquire atomic_fetch_or_acquire 744 + #define arch_atomic_fetch_or_release atomic_fetch_or_release 745 + #define arch_atomic_fetch_or_relaxed atomic_fetch_or_relaxed 746 + 808 747 #ifndef atomic_fetch_or_relaxed 809 748 #define atomic_fetch_or_acquire atomic_fetch_or 810 749 #define atomic_fetch_or_release atomic_fetch_or ··· 853 778 #endif 854 779 855 780 #endif /* atomic_fetch_or_relaxed */ 781 + 782 + #define arch_atomic_xor atomic_xor 783 + 784 + #define arch_atomic_fetch_xor atomic_fetch_xor 785 + #define arch_atomic_fetch_xor_acquire atomic_fetch_xor_acquire 786 + #define arch_atomic_fetch_xor_release atomic_fetch_xor_release 787 + #define arch_atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 856 788 857 789 #ifndef atomic_fetch_xor_relaxed 858 790 #define atomic_fetch_xor_acquire atomic_fetch_xor ··· 903 821 904 822 #endif /* atomic_fetch_xor_relaxed */ 905 823 824 + #define arch_atomic_xchg atomic_xchg 825 + #define arch_atomic_xchg_acquire atomic_xchg_acquire 826 + #define arch_atomic_xchg_release atomic_xchg_release 827 + #define arch_atomic_xchg_relaxed atomic_xchg_relaxed 828 + 906 829 #ifndef atomic_xchg_relaxed 907 830 #define atomic_xchg_acquire atomic_xchg 908 831 #define atomic_xchg_release atomic_xchg ··· 950 863 951 864 #endif /* atomic_xchg_relaxed */ 952 865 866 + #define arch_atomic_cmpxchg atomic_cmpxchg 867 + #define arch_atomic_cmpxchg_acquire atomic_cmpxchg_acquire 868 + #define arch_atomic_cmpxchg_release atomic_cmpxchg_release 869 + #define arch_atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed 870 + 953 871 #ifndef atomic_cmpxchg_relaxed 954 872 #define atomic_cmpxchg_acquire atomic_cmpxchg 955 873 #define atomic_cmpxchg_release atomic_cmpxchg ··· 996 904 #endif 997 905 998 906 #endif /* atomic_cmpxchg_relaxed */ 907 + 908 + #define arch_atomic_try_cmpxchg atomic_try_cmpxchg 909 + #define arch_atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire 910 + #define arch_atomic_try_cmpxchg_release atomic_try_cmpxchg_release 911 + #define arch_atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed 999 912 1000 913 #ifndef atomic_try_cmpxchg_relaxed 1001 914 #ifdef atomic_try_cmpxchg ··· 1099 1002 1100 1003 #endif /* atomic_try_cmpxchg_relaxed */ 1101 1004 1005 + #define arch_atomic_sub_and_test atomic_sub_and_test 1006 + 1102 1007 #ifndef atomic_sub_and_test 1103 1008 /** 1104 1009 * atomic_sub_and_test - subtract value from variable and test result ··· 1119 1020 #define atomic_sub_and_test atomic_sub_and_test 1120 1021 #endif 1121 1022 1023 + #define arch_atomic_dec_and_test atomic_dec_and_test 1024 + 1122 1025 #ifndef atomic_dec_and_test 1123 1026 /** 1124 1027 * atomic_dec_and_test - decrement and test ··· 1137 1036 } 1138 1037 #define atomic_dec_and_test atomic_dec_and_test 1139 1038 #endif 1039 + 1040 + #define arch_atomic_inc_and_test atomic_inc_and_test 1140 1041 1141 1042 #ifndef atomic_inc_and_test 1142 1043 /** ··· 1157 1054 #define atomic_inc_and_test atomic_inc_and_test 1158 1055 #endif 1159 1056 1057 + #define arch_atomic_add_negative atomic_add_negative 1058 + 1160 1059 #ifndef atomic_add_negative 1161 1060 /** 1162 1061 * atomic_add_negative - add and test if negative ··· 1176 1071 } 1177 1072 #define atomic_add_negative atomic_add_negative 1178 1073 #endif 1074 + 1075 + #define arch_atomic_fetch_add_unless atomic_fetch_add_unless 1179 1076 1180 1077 #ifndef atomic_fetch_add_unless 1181 1078 /** ··· 1204 1097 #define atomic_fetch_add_unless atomic_fetch_add_unless 1205 1098 #endif 1206 1099 1100 + #define arch_atomic_add_unless atomic_add_unless 1101 + 1207 1102 #ifndef atomic_add_unless 1208 1103 /** 1209 1104 * atomic_add_unless - add unless the number is already a given value ··· 1224 1115 #define atomic_add_unless atomic_add_unless 1225 1116 #endif 1226 1117 1118 + #define arch_atomic_inc_not_zero atomic_inc_not_zero 1119 + 1227 1120 #ifndef atomic_inc_not_zero 1228 1121 /** 1229 1122 * atomic_inc_not_zero - increment unless the number is zero ··· 1241 1130 } 1242 1131 #define atomic_inc_not_zero atomic_inc_not_zero 1243 1132 #endif 1133 + 1134 + #define arch_atomic_inc_unless_negative atomic_inc_unless_negative 1244 1135 1245 1136 #ifndef atomic_inc_unless_negative 1246 1137 static __always_inline bool ··· 1260 1147 #define atomic_inc_unless_negative atomic_inc_unless_negative 1261 1148 #endif 1262 1149 1150 + #define arch_atomic_dec_unless_positive atomic_dec_unless_positive 1151 + 1263 1152 #ifndef atomic_dec_unless_positive 1264 1153 static __always_inline bool 1265 1154 atomic_dec_unless_positive(atomic_t *v) ··· 1277 1162 } 1278 1163 #define atomic_dec_unless_positive atomic_dec_unless_positive 1279 1164 #endif 1165 + 1166 + #define arch_atomic_dec_if_positive atomic_dec_if_positive 1280 1167 1281 1168 #ifndef atomic_dec_if_positive 1282 1169 static __always_inline int ··· 1301 1184 #include <asm-generic/atomic64.h> 1302 1185 #endif 1303 1186 1187 + #define arch_atomic64_read atomic64_read 1188 + #define arch_atomic64_read_acquire atomic64_read_acquire 1189 + 1304 1190 #ifndef atomic64_read_acquire 1305 1191 static __always_inline s64 1306 1192 atomic64_read_acquire(const atomic64_t *v) ··· 1313 1193 #define atomic64_read_acquire atomic64_read_acquire 1314 1194 #endif 1315 1195 1196 + #define arch_atomic64_set atomic64_set 1197 + #define arch_atomic64_set_release atomic64_set_release 1198 + 1316 1199 #ifndef atomic64_set_release 1317 1200 static __always_inline void 1318 1201 atomic64_set_release(atomic64_t *v, s64 i) ··· 1324 1201 } 1325 1202 #define atomic64_set_release atomic64_set_release 1326 1203 #endif 1204 + 1205 + #define arch_atomic64_add atomic64_add 1206 + 1207 + #define arch_atomic64_add_return atomic64_add_return 1208 + #define arch_atomic64_add_return_acquire atomic64_add_return_acquire 1209 + #define arch_atomic64_add_return_release atomic64_add_return_release 1210 + #define arch_atomic64_add_return_relaxed atomic64_add_return_relaxed 1327 1211 1328 1212 #ifndef atomic64_add_return_relaxed 1329 1213 #define atomic64_add_return_acquire atomic64_add_return ··· 1374 1244 1375 1245 #endif /* atomic64_add_return_relaxed */ 1376 1246 1247 + #define arch_atomic64_fetch_add atomic64_fetch_add 1248 + #define arch_atomic64_fetch_add_acquire atomic64_fetch_add_acquire 1249 + #define arch_atomic64_fetch_add_release atomic64_fetch_add_release 1250 + #define arch_atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 1251 + 1377 1252 #ifndef atomic64_fetch_add_relaxed 1378 1253 #define atomic64_fetch_add_acquire atomic64_fetch_add 1379 1254 #define atomic64_fetch_add_release atomic64_fetch_add ··· 1420 1285 #endif 1421 1286 1422 1287 #endif /* atomic64_fetch_add_relaxed */ 1288 + 1289 + #define arch_atomic64_sub atomic64_sub 1290 + 1291 + #define arch_atomic64_sub_return atomic64_sub_return 1292 + #define arch_atomic64_sub_return_acquire atomic64_sub_return_acquire 1293 + #define arch_atomic64_sub_return_release atomic64_sub_return_release 1294 + #define arch_atomic64_sub_return_relaxed atomic64_sub_return_relaxed 1423 1295 1424 1296 #ifndef atomic64_sub_return_relaxed 1425 1297 #define atomic64_sub_return_acquire atomic64_sub_return ··· 1470 1328 1471 1329 #endif /* atomic64_sub_return_relaxed */ 1472 1330 1331 + #define arch_atomic64_fetch_sub atomic64_fetch_sub 1332 + #define arch_atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire 1333 + #define arch_atomic64_fetch_sub_release atomic64_fetch_sub_release 1334 + #define arch_atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 1335 + 1473 1336 #ifndef atomic64_fetch_sub_relaxed 1474 1337 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 1475 1338 #define atomic64_fetch_sub_release atomic64_fetch_sub ··· 1517 1370 1518 1371 #endif /* atomic64_fetch_sub_relaxed */ 1519 1372 1373 + #define arch_atomic64_inc atomic64_inc 1374 + 1520 1375 #ifndef atomic64_inc 1521 1376 static __always_inline void 1522 1377 atomic64_inc(atomic64_t *v) ··· 1527 1378 } 1528 1379 #define atomic64_inc atomic64_inc 1529 1380 #endif 1381 + 1382 + #define arch_atomic64_inc_return atomic64_inc_return 1383 + #define arch_atomic64_inc_return_acquire atomic64_inc_return_acquire 1384 + #define arch_atomic64_inc_return_release atomic64_inc_return_release 1385 + #define arch_atomic64_inc_return_relaxed atomic64_inc_return_relaxed 1530 1386 1531 1387 #ifndef atomic64_inc_return_relaxed 1532 1388 #ifdef atomic64_inc_return ··· 1614 1460 1615 1461 #endif /* atomic64_inc_return_relaxed */ 1616 1462 1463 + #define arch_atomic64_fetch_inc atomic64_fetch_inc 1464 + #define arch_atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire 1465 + #define arch_atomic64_fetch_inc_release atomic64_fetch_inc_release 1466 + #define arch_atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed 1467 + 1617 1468 #ifndef atomic64_fetch_inc_relaxed 1618 1469 #ifdef atomic64_fetch_inc 1619 1470 #define atomic64_fetch_inc_acquire atomic64_fetch_inc ··· 1700 1541 1701 1542 #endif /* atomic64_fetch_inc_relaxed */ 1702 1543 1544 + #define arch_atomic64_dec atomic64_dec 1545 + 1703 1546 #ifndef atomic64_dec 1704 1547 static __always_inline void 1705 1548 atomic64_dec(atomic64_t *v) ··· 1710 1549 } 1711 1550 #define atomic64_dec atomic64_dec 1712 1551 #endif 1552 + 1553 + #define arch_atomic64_dec_return atomic64_dec_return 1554 + #define arch_atomic64_dec_return_acquire atomic64_dec_return_acquire 1555 + #define arch_atomic64_dec_return_release atomic64_dec_return_release 1556 + #define arch_atomic64_dec_return_relaxed atomic64_dec_return_relaxed 1713 1557 1714 1558 #ifndef atomic64_dec_return_relaxed 1715 1559 #ifdef atomic64_dec_return ··· 1797 1631 1798 1632 #endif /* atomic64_dec_return_relaxed */ 1799 1633 1634 + #define arch_atomic64_fetch_dec atomic64_fetch_dec 1635 + #define arch_atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire 1636 + #define arch_atomic64_fetch_dec_release atomic64_fetch_dec_release 1637 + #define arch_atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed 1638 + 1800 1639 #ifndef atomic64_fetch_dec_relaxed 1801 1640 #ifdef atomic64_fetch_dec 1802 1641 #define atomic64_fetch_dec_acquire atomic64_fetch_dec ··· 1883 1712 1884 1713 #endif /* atomic64_fetch_dec_relaxed */ 1885 1714 1715 + #define arch_atomic64_and atomic64_and 1716 + 1717 + #define arch_atomic64_fetch_and atomic64_fetch_and 1718 + #define arch_atomic64_fetch_and_acquire atomic64_fetch_and_acquire 1719 + #define arch_atomic64_fetch_and_release atomic64_fetch_and_release 1720 + #define arch_atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 1721 + 1886 1722 #ifndef atomic64_fetch_and_relaxed 1887 1723 #define atomic64_fetch_and_acquire atomic64_fetch_and 1888 1724 #define atomic64_fetch_and_release atomic64_fetch_and ··· 1932 1754 1933 1755 #endif /* atomic64_fetch_and_relaxed */ 1934 1756 1757 + #define arch_atomic64_andnot atomic64_andnot 1758 + 1935 1759 #ifndef atomic64_andnot 1936 1760 static __always_inline void 1937 1761 atomic64_andnot(s64 i, atomic64_t *v) ··· 1942 1762 } 1943 1763 #define atomic64_andnot atomic64_andnot 1944 1764 #endif 1765 + 1766 + #define arch_atomic64_fetch_andnot atomic64_fetch_andnot 1767 + #define arch_atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire 1768 + #define arch_atomic64_fetch_andnot_release atomic64_fetch_andnot_release 1769 + #define arch_atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed 1945 1770 1946 1771 #ifndef atomic64_fetch_andnot_relaxed 1947 1772 #ifdef atomic64_fetch_andnot ··· 2029 1844 2030 1845 #endif /* atomic64_fetch_andnot_relaxed */ 2031 1846 1847 + #define arch_atomic64_or atomic64_or 1848 + 1849 + #define arch_atomic64_fetch_or atomic64_fetch_or 1850 + #define arch_atomic64_fetch_or_acquire atomic64_fetch_or_acquire 1851 + #define arch_atomic64_fetch_or_release atomic64_fetch_or_release 1852 + #define arch_atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 1853 + 2032 1854 #ifndef atomic64_fetch_or_relaxed 2033 1855 #define atomic64_fetch_or_acquire atomic64_fetch_or 2034 1856 #define atomic64_fetch_or_release atomic64_fetch_or ··· 2077 1885 #endif 2078 1886 2079 1887 #endif /* atomic64_fetch_or_relaxed */ 1888 + 1889 + #define arch_atomic64_xor atomic64_xor 1890 + 1891 + #define arch_atomic64_fetch_xor atomic64_fetch_xor 1892 + #define arch_atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire 1893 + #define arch_atomic64_fetch_xor_release atomic64_fetch_xor_release 1894 + #define arch_atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 2080 1895 2081 1896 #ifndef atomic64_fetch_xor_relaxed 2082 1897 #define atomic64_fetch_xor_acquire atomic64_fetch_xor ··· 2127 1928 2128 1929 #endif /* atomic64_fetch_xor_relaxed */ 2129 1930 1931 + #define arch_atomic64_xchg atomic64_xchg 1932 + #define arch_atomic64_xchg_acquire atomic64_xchg_acquire 1933 + #define arch_atomic64_xchg_release atomic64_xchg_release 1934 + #define arch_atomic64_xchg_relaxed atomic64_xchg_relaxed 1935 + 2130 1936 #ifndef atomic64_xchg_relaxed 2131 1937 #define atomic64_xchg_acquire atomic64_xchg 2132 1938 #define atomic64_xchg_release atomic64_xchg ··· 2174 1970 2175 1971 #endif /* atomic64_xchg_relaxed */ 2176 1972 1973 + #define arch_atomic64_cmpxchg atomic64_cmpxchg 1974 + #define arch_atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire 1975 + #define arch_atomic64_cmpxchg_release atomic64_cmpxchg_release 1976 + #define arch_atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed 1977 + 2177 1978 #ifndef atomic64_cmpxchg_relaxed 2178 1979 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 2179 1980 #define atomic64_cmpxchg_release atomic64_cmpxchg ··· 2220 2011 #endif 2221 2012 2222 2013 #endif /* atomic64_cmpxchg_relaxed */ 2014 + 2015 + #define arch_atomic64_try_cmpxchg atomic64_try_cmpxchg 2016 + #define arch_atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire 2017 + #define arch_atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release 2018 + #define arch_atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed 2223 2019 2224 2020 #ifndef atomic64_try_cmpxchg_relaxed 2225 2021 #ifdef atomic64_try_cmpxchg ··· 2323 2109 2324 2110 #endif /* atomic64_try_cmpxchg_relaxed */ 2325 2111 2112 + #define arch_atomic64_sub_and_test atomic64_sub_and_test 2113 + 2326 2114 #ifndef atomic64_sub_and_test 2327 2115 /** 2328 2116 * atomic64_sub_and_test - subtract value from variable and test result ··· 2343 2127 #define atomic64_sub_and_test atomic64_sub_and_test 2344 2128 #endif 2345 2129 2130 + #define arch_atomic64_dec_and_test atomic64_dec_and_test 2131 + 2346 2132 #ifndef atomic64_dec_and_test 2347 2133 /** 2348 2134 * atomic64_dec_and_test - decrement and test ··· 2361 2143 } 2362 2144 #define atomic64_dec_and_test atomic64_dec_and_test 2363 2145 #endif 2146 + 2147 + #define arch_atomic64_inc_and_test atomic64_inc_and_test 2364 2148 2365 2149 #ifndef atomic64_inc_and_test 2366 2150 /** ··· 2381 2161 #define atomic64_inc_and_test atomic64_inc_and_test 2382 2162 #endif 2383 2163 2164 + #define arch_atomic64_add_negative atomic64_add_negative 2165 + 2384 2166 #ifndef atomic64_add_negative 2385 2167 /** 2386 2168 * atomic64_add_negative - add and test if negative ··· 2400 2178 } 2401 2179 #define atomic64_add_negative atomic64_add_negative 2402 2180 #endif 2181 + 2182 + #define arch_atomic64_fetch_add_unless atomic64_fetch_add_unless 2403 2183 2404 2184 #ifndef atomic64_fetch_add_unless 2405 2185 /** ··· 2428 2204 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 2429 2205 #endif 2430 2206 2207 + #define arch_atomic64_add_unless atomic64_add_unless 2208 + 2431 2209 #ifndef atomic64_add_unless 2432 2210 /** 2433 2211 * atomic64_add_unless - add unless the number is already a given value ··· 2448 2222 #define atomic64_add_unless atomic64_add_unless 2449 2223 #endif 2450 2224 2225 + #define arch_atomic64_inc_not_zero atomic64_inc_not_zero 2226 + 2451 2227 #ifndef atomic64_inc_not_zero 2452 2228 /** 2453 2229 * atomic64_inc_not_zero - increment unless the number is zero ··· 2465 2237 } 2466 2238 #define atomic64_inc_not_zero atomic64_inc_not_zero 2467 2239 #endif 2240 + 2241 + #define arch_atomic64_inc_unless_negative atomic64_inc_unless_negative 2468 2242 2469 2243 #ifndef atomic64_inc_unless_negative 2470 2244 static __always_inline bool ··· 2484 2254 #define atomic64_inc_unless_negative atomic64_inc_unless_negative 2485 2255 #endif 2486 2256 2257 + #define arch_atomic64_dec_unless_positive atomic64_dec_unless_positive 2258 + 2487 2259 #ifndef atomic64_dec_unless_positive 2488 2260 static __always_inline bool 2489 2261 atomic64_dec_unless_positive(atomic64_t *v) ··· 2501 2269 } 2502 2270 #define atomic64_dec_unless_positive atomic64_dec_unless_positive 2503 2271 #endif 2272 + 2273 + #define arch_atomic64_dec_if_positive atomic64_dec_if_positive 2504 2274 2505 2275 #ifndef atomic64_dec_if_positive 2506 2276 static __always_inline s64 ··· 2522 2288 #endif 2523 2289 2524 2290 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 2525 - // 1fac0941c79bf0ae100723cc2ac9b94061f0b67a 2291 + // 9d95b56f98d82a2a26c7b79ccdd0c47572d50a6f

+1 -1

include/linux/compiler.h

··· 123 123 #ifdef CONFIG_DEBUG_ENTRY 124 124 /* Begin/end of an instrumentation safe region */ 125 125 #define instrumentation_begin() ({ \ 126 - asm volatile("%c0:\n\t" \ 126 + asm volatile("%c0: nop\n\t" \ 127 127 ".pushsection .discard.instr_begin\n\t" \ 128 128 ".long %c0b - .\n\t" \ 129 129 ".popsection\n\t" : : "i" (__COUNTER__)); \

+25 -7

kernel/rcu/tree.c

··· 250 250 * next idle sojourn. 251 251 */ 252 252 rcu_dynticks_task_trace_enter(); // Before ->dynticks update! 253 - seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 253 + seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 254 254 // RCU is no longer watching. Better be in extended quiescent state! 255 255 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && 256 256 (seq & RCU_DYNTICK_CTRL_CTR)); ··· 274 274 * and we also must force ordering with the next RCU read-side 275 275 * critical section. 276 276 */ 277 - seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 277 + seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 278 278 // RCU is now watching. Better not be in an extended quiescent state! 279 279 rcu_dynticks_task_trace_exit(); // After ->dynticks update! 280 280 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && 281 281 !(seq & RCU_DYNTICK_CTRL_CTR)); 282 282 if (seq & RCU_DYNTICK_CTRL_MASK) { 283 - atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); 283 + arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); 284 284 smp_mb__after_atomic(); /* _exit after clearing mask. */ 285 285 } 286 286 } ··· 313 313 { 314 314 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 315 315 316 - return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR); 316 + return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR); 317 317 } 318 318 319 319 /* ··· 633 633 do_nocb_deferred_wakeup(rdp); 634 634 rcu_prepare_for_idle(); 635 635 rcu_preempt_deferred_qs(current); 636 + 637 + // instrumentation for the noinstr rcu_dynticks_eqs_enter() 638 + instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); 639 + 636 640 instrumentation_end(); 637 641 WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ 638 642 // RCU is watching here ... ··· 696 692 { 697 693 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 698 694 695 + instrumentation_begin(); 699 696 /* 700 697 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. 701 698 * (We are exiting an NMI handler, so RCU better be paying attention ··· 710 705 * leave it in non-RCU-idle state. 711 706 */ 712 707 if (rdp->dynticks_nmi_nesting != 1) { 713 - instrumentation_begin(); 714 708 trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, 715 709 atomic_read(&rdp->dynticks)); 716 710 WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ ··· 718 714 return; 719 715 } 720 716 721 - instrumentation_begin(); 722 717 /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ 723 718 trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks)); 724 719 WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ 725 720 726 721 if (!in_nmi()) 727 722 rcu_prepare_for_idle(); 723 + 724 + // instrumentation for the noinstr rcu_dynticks_eqs_enter() 725 + instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); 728 726 instrumentation_end(); 729 727 730 728 // RCU is watching here ... ··· 844 838 rcu_dynticks_eqs_exit(); 845 839 // ... but is watching here. 846 840 instrumentation_begin(); 841 + 842 + // instrumentation for the noinstr rcu_dynticks_eqs_exit() 843 + instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); 844 + 847 845 rcu_cleanup_after_idle(); 848 846 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); 849 847 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); ··· 993 983 if (!in_nmi()) 994 984 rcu_cleanup_after_idle(); 995 985 986 + instrumentation_begin(); 987 + // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() 988 + instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks)); 989 + // instrumentation for the noinstr rcu_dynticks_eqs_exit() 990 + instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); 991 + 996 992 incby = 1; 997 993 } else if (!in_nmi()) { 998 994 instrumentation_begin(); 999 995 rcu_irq_enter_check_tick(); 1000 996 instrumentation_end(); 997 + } else { 998 + instrumentation_begin(); 1001 999 } 1002 - instrumentation_begin(); 1000 + 1003 1001 trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), 1004 1002 rdp->dynticks_nmi_nesting, 1005 1003 rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));

+31

scripts/atomic/gen-atomic-fallback.sh

··· 58 58 EOF 59 59 } 60 60 61 + gen_proto_order_variant() 62 + { 63 + local meta="$1"; shift 64 + local pfx="$1"; shift 65 + local name="$1"; shift 66 + local sfx="$1"; shift 67 + local order="$1"; shift 68 + local arch="$1" 69 + local atomic="$2" 70 + 71 + local basename="${arch}${atomic}_${pfx}${name}${sfx}" 72 + 73 + printf "#define arch_${basename}${order} ${basename}${order}\n" 74 + } 75 + 61 76 #gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...) 62 77 gen_proto_order_variants() 63 78 { ··· 86 71 local basename="${arch}${atomic}_${pfx}${name}${sfx}" 87 72 88 73 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" 74 + 75 + if [ -z "$arch" ]; then 76 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" 77 + 78 + if meta_has_acquire "${meta}"; then 79 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" 80 + fi 81 + if meta_has_release "${meta}"; then 82 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" 83 + fi 84 + if meta_has_relaxed "${meta}"; then 85 + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" 86 + fi 87 + 88 + echo "" 89 + fi 89 90 90 91 # If we don't have relaxed atomics, then we don't bother with ordering fallbacks 91 92 # read_acquire and set_release need to be templated, though

Configure Feed

Configure Feed