tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

srcu: Create an srcu_expedite_current() function

This commit creates an srcu_expedite_current() function that expedites
the current (and possibly the next) SRCU grace period for the specified
srcu_struct structure. This functionality will be inherited by RCU
Tasks Trace courtesy of its mapping to SRCU fast.

If the current SRCU grace period is already waiting, that wait will
complete before the expediting takes effect. If there is no SRCU grace
period in flight, this function might well create one.

[ paulmck: Apply Zqiang feedback for PREEMPT_RT use. ]

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

authored by

Paul E. McKenney and committed by
Frederic Weisbecker 34dc27f0 58ac42f2

+67
+1
include/linux/srcutiny.h
··· 103 103 synchronize_srcu(ssp); 104 104 } 105 105 106 + static inline void srcu_expedite_current(struct srcu_struct *ssp) { } 106 107 #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0) 107 108 #define srcu_check_read_flavor_force(ssp, read_flavor) do { } while (0) 108 109
+8
include/linux/srcutree.h
··· 42 42 struct timer_list delay_work; /* Delay for CB invoking */ 43 43 struct work_struct work; /* Context for CB invoking. */ 44 44 struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */ 45 + struct rcu_head srcu_ec_head; /* For srcu_expedite_current() use. */ 46 + int srcu_ec_state; /* State for srcu_expedite_current(). */ 45 47 struct srcu_node *mynode; /* Leaf srcu_node. */ 46 48 unsigned long grpmask; /* Mask for leaf srcu_node */ 47 49 /* ->srcu_data_have_cbs[]. */ ··· 137 135 #define SRCU_STATE_SCAN1 1 138 136 #define SRCU_STATE_SCAN2 2 139 137 138 + /* Values for srcu_expedite_current() state (->srcu_ec_state). */ 139 + #define SRCU_EC_IDLE 0 140 + #define SRCU_EC_PENDING 1 141 + #define SRCU_EC_REPOST 2 142 + 140 143 /* 141 144 * Values for initializing gp sequence fields. Higher values allow wrap arounds to 142 145 * occur earlier. ··· 217 210 int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); 218 211 void synchronize_srcu_expedited(struct srcu_struct *ssp); 219 212 void srcu_barrier(struct srcu_struct *ssp); 213 + void srcu_expedite_current(struct srcu_struct *ssp); 220 214 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf); 221 215 222 216 // Converts a per-CPU pointer to an ->srcu_ctrs[] array element to that
+58
kernel/rcu/srcutree.c
··· 1688 1688 } 1689 1689 EXPORT_SYMBOL_GPL(srcu_barrier); 1690 1690 1691 + /* Callback for srcu_expedite_current() usage. */ 1692 + static void srcu_expedite_current_cb(struct rcu_head *rhp) 1693 + { 1694 + unsigned long flags; 1695 + bool needcb = false; 1696 + struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head); 1697 + 1698 + spin_lock_irqsave_sdp_contention(sdp, &flags); 1699 + if (sdp->srcu_ec_state == SRCU_EC_IDLE) { 1700 + WARN_ON_ONCE(1); 1701 + } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) { 1702 + sdp->srcu_ec_state = SRCU_EC_IDLE; 1703 + } else { 1704 + WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST); 1705 + sdp->srcu_ec_state = SRCU_EC_PENDING; 1706 + needcb = true; 1707 + } 1708 + spin_unlock_irqrestore_rcu_node(sdp, flags); 1709 + // If needed, requeue ourselves as an expedited SRCU callback. 1710 + if (needcb) 1711 + __call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); 1712 + } 1713 + 1714 + /** 1715 + * srcu_expedite_current - Expedite the current SRCU grace period 1716 + * @ssp: srcu_struct to expedite. 1717 + * 1718 + * Cause the current SRCU grace period to become expedited. The grace 1719 + * period following the current one might also be expedited. If there is 1720 + * no current grace period, one might be created. If the current grace 1721 + * period is currently sleeping, that sleep will complete before expediting 1722 + * will take effect. 1723 + */ 1724 + void srcu_expedite_current(struct srcu_struct *ssp) 1725 + { 1726 + unsigned long flags; 1727 + bool needcb = false; 1728 + struct srcu_data *sdp; 1729 + 1730 + migrate_disable(); 1731 + sdp = this_cpu_ptr(ssp->sda); 1732 + spin_lock_irqsave_sdp_contention(sdp, &flags); 1733 + if (sdp->srcu_ec_state == SRCU_EC_IDLE) { 1734 + sdp->srcu_ec_state = SRCU_EC_PENDING; 1735 + needcb = true; 1736 + } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) { 1737 + sdp->srcu_ec_state = SRCU_EC_REPOST; 1738 + } else { 1739 + WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST); 1740 + } 1741 + spin_unlock_irqrestore_rcu_node(sdp, flags); 1742 + // If needed, queue an expedited SRCU callback. 1743 + if (needcb) 1744 + __call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); 1745 + migrate_enable(); 1746 + } 1747 + EXPORT_SYMBOL_GPL(srcu_expedite_current); 1748 + 1691 1749 /** 1692 1750 * srcu_batches_completed - return batches completed. 1693 1751 * @ssp: srcu_struct on which to report batch completion.