workqueue: Move nr_active handling into helpers

__queue_work(), pwq_dec_nr_in_flight() and wq_adjust_max_active() were
open-coding nr_active handling, which is fine given that the operations are
trivial. However, the planned unbound nr_active update will make them more
complicated, so let's move them into helpers.

- pwq_tryinc_nr_active() is added. It increments nr_active if under
max_active limit and return a boolean indicating whether inc was
successful. Note that the function is structured to accommodate future
changes. __queue_work() is updated to use the new helper.

- pwq_activate_first_inactive() is updated to use pwq_tryinc_nr_active() and
thus no longer assumes that nr_active is under max_active and returns a
boolean to indicate whether a work item has been activated.

- wq_adjust_max_active() no longer tests directly whether a work item can be
activated. Instead, it's updated to use the return value of
pwq_activate_first_inactive() to tell whether a work item has been
activated.

- nr_active decrement and activating the first inactive work item is
factored into pwq_dec_nr_active().

v3: - WARN_ON_ONCE(!WORK_STRUCT_INACTIVE) added to __pwq_activate_work() as
now we're calling the function unconditionally from
pwq_activate_first_inactive().

v2: - wq->max_active now uses WRITE/READ_ONCE() as suggested by Lai.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Lai Jiangshan <jiangshanlai@gmail.com>

Tejun Heo 2 years ago 1c270b79 4c638030

+67 -19

1 changed file

expand all

kernel

workqueue.c

+67 -19

kernel/workqueue.c

··· 1468 1468 static void __pwq_activate_work(struct pool_workqueue *pwq, 1469 1469 struct work_struct *work) 1470 1470 { 1471 + unsigned long *wdb = work_data_bits(work); 1472 + 1473 + WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE)); 1471 1474 trace_workqueue_activate_work(work); 1472 1475 if (list_empty(&pwq->pool->worklist)) 1473 1476 pwq->pool->watchdog_ts = jiffies; 1474 1477 move_linked_works(work, &pwq->pool->worklist, NULL); 1475 - __clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work)); 1478 + __clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb); 1476 1479 } 1477 1480 1478 1481 /** ··· 1500 1497 return true; 1501 1498 } 1502 1499 1503 - static void pwq_activate_first_inactive(struct pool_workqueue *pwq) 1500 + /** 1501 + * pwq_tryinc_nr_active - Try to increment nr_active for a pwq 1502 + * @pwq: pool_workqueue of interest 1503 + * 1504 + * Try to increment nr_active for @pwq. Returns %true if an nr_active count is 1505 + * successfully obtained. %false otherwise. 1506 + */ 1507 + static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq) 1504 1508 { 1505 - struct work_struct *work = list_first_entry(&pwq->inactive_works, 1506 - struct work_struct, entry); 1509 + struct workqueue_struct *wq = pwq->wq; 1510 + struct worker_pool *pool = pwq->pool; 1511 + bool obtained; 1507 1512 1508 - pwq_activate_work(pwq, work); 1513 + lockdep_assert_held(&pool->lock); 1514 + 1515 + obtained = pwq->nr_active < READ_ONCE(wq->max_active); 1516 + 1517 + if (obtained) 1518 + pwq->nr_active++; 1519 + return obtained; 1520 + } 1521 + 1522 + /** 1523 + * pwq_activate_first_inactive - Activate the first inactive work item on a pwq 1524 + * @pwq: pool_workqueue of interest 1525 + * 1526 + * Activate the first inactive work item of @pwq if available and allowed by 1527 + * max_active limit. 1528 + * 1529 + * Returns %true if an inactive work item has been activated. %false if no 1530 + * inactive work item is found or max_active limit is reached. 1531 + */ 1532 + static bool pwq_activate_first_inactive(struct pool_workqueue *pwq) 1533 + { 1534 + struct work_struct *work = 1535 + list_first_entry_or_null(&pwq->inactive_works, 1536 + struct work_struct, entry); 1537 + 1538 + if (work && pwq_tryinc_nr_active(pwq)) { 1539 + __pwq_activate_work(pwq, work); 1540 + return true; 1541 + } else { 1542 + return false; 1543 + } 1544 + } 1545 + 1546 + /** 1547 + * pwq_dec_nr_active - Retire an active count 1548 + * @pwq: pool_workqueue of interest 1549 + * 1550 + * Decrement @pwq's nr_active and try to activate the first inactive work item. 1551 + */ 1552 + static void pwq_dec_nr_active(struct pool_workqueue *pwq) 1553 + { 1554 + struct worker_pool *pool = pwq->pool; 1555 + 1556 + lockdep_assert_held(&pool->lock); 1557 + 1558 + pwq->nr_active--; 1559 + pwq_activate_first_inactive(pwq); 1509 1560 } 1510 1561 1511 1562 /** ··· 1577 1520 { 1578 1521 int color = get_work_color(work_data); 1579 1522 1580 - if (!(work_data & WORK_STRUCT_INACTIVE)) { 1581 - pwq->nr_active--; 1582 - if (!list_empty(&pwq->inactive_works)) { 1583 - /* one down, submit an inactive one */ 1584 - if (pwq->nr_active < READ_ONCE(pwq->wq->max_active)) 1585 - pwq_activate_first_inactive(pwq); 1586 - } 1587 - } 1523 + if (!(work_data & WORK_STRUCT_INACTIVE)) 1524 + pwq_dec_nr_active(pwq); 1588 1525 1589 1526 pwq->nr_in_flight[color]--; 1590 1527 ··· 1880 1829 * @work must also queue behind existing inactive work items to maintain 1881 1830 * ordering when max_active changes. See wq_adjust_max_active(). 1882 1831 */ 1883 - if (list_empty(&pwq->inactive_works) && 1884 - pwq->nr_active < READ_ONCE(pwq->wq->max_active)) { 1832 + if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq)) { 1885 1833 if (list_empty(&pool->worklist)) 1886 1834 pool->watchdog_ts = jiffies; 1887 1835 1888 1836 trace_workqueue_activate_work(work); 1889 - pwq->nr_active++; 1890 1837 insert_work(pwq, work, &pool->worklist, work_flags); 1891 1838 kick_pool(pool); 1892 1839 } else { ··· 4736 4687 /* this function can be called during early boot w/ irq disabled */ 4737 4688 raw_spin_lock_irqsave(&pwq->pool->lock, flags); 4738 4689 4739 - while (!list_empty(&pwq->inactive_works) && 4740 - pwq->nr_active < wq->max_active) 4741 - pwq_activate_first_inactive(pwq); 4690 + while (pwq_activate_first_inactive(pwq)) 4691 + ; 4742 4692 4743 4693 kick_pool(pwq->pool); 4744 4694

Configure Feed

Configure Feed