sched_ext: Make explicit scx_task_iter_relock() calls unnecessary

During tasks iteration, the locks can be dropped using
scx_task_iter_unlock() to perform e.g. sleepable allocations. Afterwards,
scx_task_iter_relock() has to be called prior to other iteration operations,
which is error-prone. This can be easily automated by tracking whether
scx_tasks_lock is held in scx_task_iter and re-acquiring when necessary. It
already tracks whether the task's rq is locked after all.

- Add scx_task_iter->list_locked which remembers whether scx_tasks_lock is
held.

- Rename scx_task_iter->locked to scx_task_iter->locked_task to better
distinguish it from ->list_locked.

- Replace scx_task_iter_relock() with __scx_task_iter_maybe_relock() which
is automatically called by scx_task_iter_next() and scx_task_iter_stop().

- Drop explicit scx_task_iter_relock() calls.

The resulting behavior should be equivalent.

Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Andrea Righi <arighi@nvidia.com>

Tejun Heo 9 months ago b7975c48 de68c051

+23 -20

1 changed file

expand all

kernel

sched

ext.c

+23 -20

kernel/sched/ext.c

··· 1488 1488 */ 1489 1489 struct scx_task_iter { 1490 1490 struct sched_ext_entity cursor; 1491 - struct task_struct *locked; 1491 + struct task_struct *locked_task; 1492 1492 struct rq *rq; 1493 1493 struct rq_flags rf; 1494 1494 u32 cnt; 1495 + bool list_locked; 1495 1496 }; 1496 1497 1497 1498 /** ··· 1520 1519 1521 1520 iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; 1522 1521 list_add(&iter->cursor.tasks_node, &scx_tasks); 1523 - iter->locked = NULL; 1522 + iter->locked_task = NULL; 1524 1523 iter->cnt = 0; 1524 + iter->list_locked = true; 1525 1525 } 1526 1526 1527 1527 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) 1528 1528 { 1529 - if (iter->locked) { 1530 - task_rq_unlock(iter->rq, iter->locked, &iter->rf); 1531 - iter->locked = NULL; 1529 + if (iter->locked_task) { 1530 + task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); 1531 + iter->locked_task = NULL; 1532 1532 } 1533 1533 } 1534 1534 ··· 1539 1537 * 1540 1538 * If @iter is in the middle of a locked iteration, it may be locking the rq of 1541 1539 * the task currently being visited in addition to scx_tasks_lock. Unlock both. 1542 - * This function can be safely called anytime during an iteration. 1540 + * This function can be safely called anytime during an iteration. The next 1541 + * iterator operation will automatically restore the necessary locking. 1543 1542 */ 1544 1543 static void scx_task_iter_unlock(struct scx_task_iter *iter) 1545 1544 { 1546 1545 __scx_task_iter_rq_unlock(iter); 1547 - spin_unlock_irq(&scx_tasks_lock); 1546 + if (iter->list_locked) { 1547 + iter->list_locked = false; 1548 + spin_unlock_irq(&scx_tasks_lock); 1549 + } 1548 1550 } 1549 1551 1550 - /** 1551 - * scx_task_iter_relock - Lock scx_tasks_lock released by scx_task_iter_unlock() 1552 - * @iter: iterator to re-lock 1553 - * 1554 - * Re-lock scx_tasks_lock unlocked by scx_task_iter_unlock(). Note that it 1555 - * doesn't re-lock the rq lock. Must be called before other iterator operations. 1556 - */ 1557 - static void scx_task_iter_relock(struct scx_task_iter *iter) 1552 + static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) 1558 1553 { 1559 - spin_lock_irq(&scx_tasks_lock); 1554 + if (!iter->list_locked) { 1555 + spin_lock_irq(&scx_tasks_lock); 1556 + iter->list_locked = true; 1557 + } 1560 1558 } 1561 1559 1562 1560 /** ··· 1569 1567 */ 1570 1568 static void scx_task_iter_stop(struct scx_task_iter *iter) 1571 1569 { 1570 + __scx_task_iter_maybe_relock(iter); 1572 1571 list_del_init(&iter->cursor.tasks_node); 1573 1572 scx_task_iter_unlock(iter); 1574 1573 } ··· 1587 1584 struct list_head *cursor = &iter->cursor.tasks_node; 1588 1585 struct sched_ext_entity *pos; 1589 1586 1587 + __scx_task_iter_maybe_relock(iter); 1588 + 1590 1589 if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { 1591 1590 scx_task_iter_unlock(iter); 1592 1591 cond_resched(); 1593 - scx_task_iter_relock(iter); 1592 + __scx_task_iter_maybe_relock(iter); 1594 1593 } 1595 1594 1596 1595 list_for_each_entry(pos, cursor, tasks_node) { ··· 1655 1650 return NULL; 1656 1651 1657 1652 iter->rq = task_rq_lock(p, &iter->rf); 1658 - iter->locked = p; 1653 + iter->locked_task = p; 1659 1654 1660 1655 return p; 1661 1656 } ··· 5718 5713 ret = scx_init_task(p, task_group(p), false); 5719 5714 if (ret) { 5720 5715 put_task_struct(p); 5721 - scx_task_iter_relock(&sti); 5722 5716 scx_task_iter_stop(&sti); 5723 5717 scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", 5724 5718 ret, p->comm, p->pid); ··· 5727 5723 scx_set_task_state(p, SCX_TASK_READY); 5728 5724 5729 5725 put_task_struct(p); 5730 - scx_task_iter_relock(&sti); 5731 5726 } 5732 5727 scx_task_iter_stop(&sti); 5733 5728 scx_cgroup_unlock();

Configure Feed

Configure Feed