Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
rcu/nocb: Extract nocb_bypass_needs_flush() to reduce duplication
The bypass flush decision logic is duplicated in rcu_nocb_try_bypass()
and nocb_gp_wait() with similar conditions.
This commit therefore extracts the functionality into a common helper
function nocb_bypass_needs_flush() improving the code readability.
A flush_faster parameter is added to controlling the flushing thresholds
and timeouts. This design was in the original commit d1b222c6be1f
("rcu/nocb: Add bypass callback queueing") to avoid having the GP
kthread aggressively flush the bypass queue.
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
+37
-14
+37
-14
kernel/rcu/tree_nocb.h
+37
-14
kernel/rcu/tree_nocb.h
···
379
379
}
380
380
381
381
/*
382
+
* Determine if the bypass queue needs to be flushed based on time and size.
383
+
* For lazy-only bypass queues, use the lazy flush timeout; otherwise flush
384
+
* based on jiffy advancement. The flush_faster controls flush aggressiveness.
385
+
*/
386
+
static bool nocb_bypass_needs_flush(struct rcu_data *rdp, long bypass_ncbs,
387
+
long lazy_ncbs, unsigned long j,
388
+
bool flush_faster)
389
+
{
390
+
bool bypass_is_lazy;
391
+
unsigned long bypass_first;
392
+
unsigned long flush_timeout;
393
+
long qhimark_thresh;
394
+
395
+
if (!bypass_ncbs)
396
+
return false;
397
+
398
+
qhimark_thresh = flush_faster ? qhimark : 2 * qhimark;
399
+
if (bypass_ncbs >= qhimark_thresh)
400
+
return true;
401
+
402
+
bypass_first = READ_ONCE(rdp->nocb_bypass_first);
403
+
bypass_is_lazy = (bypass_ncbs == lazy_ncbs);
404
+
405
+
if (bypass_is_lazy)
406
+
flush_timeout = rcu_get_jiffies_lazy_flush();
407
+
else
408
+
flush_timeout = flush_faster ? 0 : 1;
409
+
410
+
return time_after(j, bypass_first + flush_timeout);
411
+
}
412
+
413
+
/*
382
414
* See whether it is appropriate to use the ->nocb_bypass list in order
383
415
* to control contention on ->nocb_lock. A limited number of direct
384
416
* enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
···
436
404
unsigned long cur_gp_seq;
437
405
unsigned long j = jiffies;
438
406
long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
439
-
bool bypass_is_lazy = (ncbs == READ_ONCE(rdp->lazy_len));
407
+
long lazy_len = READ_ONCE(rdp->lazy_len);
408
+
bool bypass_is_lazy = (ncbs == lazy_len);
440
409
441
410
lockdep_assert_irqs_disabled();
442
411
···
489
456
490
457
// If ->nocb_bypass has been used too long or is too full,
491
458
// flush ->nocb_bypass to ->cblist.
492
-
if ((ncbs && !bypass_is_lazy && j != READ_ONCE(rdp->nocb_bypass_first)) ||
493
-
(ncbs && bypass_is_lazy &&
494
-
(time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()))) ||
495
-
ncbs >= qhimark) {
459
+
if (nocb_bypass_needs_flush(rdp, ncbs, lazy_len, j, true)) {
496
460
rcu_nocb_lock(rdp);
497
461
*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
498
462
···
703
673
bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
704
674
lazy_ncbs = READ_ONCE(rdp->lazy_len);
705
675
706
-
if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) &&
707
-
(time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()) ||
708
-
bypass_ncbs > 2 * qhimark)) {
709
-
flush_bypass = true;
710
-
} else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) &&
711
-
(time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||
712
-
bypass_ncbs > 2 * qhimark)) {
713
-
flush_bypass = true;
714
-
} else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
676
+
flush_bypass = nocb_bypass_needs_flush(rdp, bypass_ncbs, lazy_ncbs, j, false);
677
+
if (!flush_bypass && !bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
715
678
rcu_nocb_unlock_irqrestore(rdp, flags);
716
679
continue; /* No callbacks here, try next. */
717
680
}