sched/topology: Switch to assigning "sd->shared" from s_data

Use the "sched_domain_shared" object allocated in s_data for
"sd->shared" assignments. Assign "sd->shared" for the topmost
SD_SHARE_LLC domain before degeneration and rely on the degeneration
path to correctly pass down the shared object to "sd_llc".

sd_degenerate_parent() ensures degenerating domains must have the same
sched_domain_span() which ensures 1:1 passing down of the shared object.
If the topmost SD_SHARE_LLC domain degenerates, the shared object is
freed from destroy_sched_domain() when the last reference is dropped.

claim_allocations() NULLs out the objects that have been assigned as
"sd->shared" and the unassigned ones are freed from the __sds_free()
path.

To keep all the claim_allocations() bits in one place,
claim_allocations() has been extended to accept "s_data" and iterate the
domains internally to free both "sched_domain_shared" and the
per-topology-level data for the particular CPU in one place.

Post cpu_attach_domain(), all reclaims of "sd->shared" are handled via
call_rcu() on the sched_domain object via destroy_sched_domains_rcu().

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Tested-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Link: https://patch.msgid.link/20260312044434.1974-5-kprateek.nayak@amd.com

authored by

K Prateek Nayak and committed by

Peter Zijlstra 2 months ago bb7a5e44 1cc8a33c

+44 -29

1 changed file

expand all

kernel

sched

topology.c

+44 -29

kernel/sched/topology.c

··· 685 685 if (sd) { 686 686 id = cpumask_first(sched_domain_span(sd)); 687 687 size = cpumask_weight(sched_domain_span(sd)); 688 + 689 + /* If sd_llc exists, sd_llc_shared should exist too. */ 690 + WARN_ON_ONCE(!sd->shared); 688 691 sds = sd->shared; 689 692 } 690 693 ··· 735 732 736 733 if (sd_parent_degenerate(tmp, parent)) { 737 734 tmp->parent = parent->parent; 735 + 736 + /* Pick reference to parent->shared. */ 737 + if (parent->shared) { 738 + WARN_ON_ONCE(tmp->shared); 739 + tmp->shared = parent->shared; 740 + parent->shared = NULL; 741 + } 738 742 739 743 if (parent->parent) { 740 744 parent->parent->child = tmp; ··· 1596 1586 * sched_group structure so that the subsequent __free_domain_allocs() 1597 1587 * will not free the data we're using. 1598 1588 */ 1599 - static void claim_allocations(int cpu, struct sched_domain *sd) 1589 + static void claim_allocations(int cpu, struct s_data *d) 1600 1590 { 1601 - struct sd_data *sdd = sd->private; 1591 + struct sched_domain *sd; 1602 1592 1603 - WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); 1604 - *per_cpu_ptr(sdd->sd, cpu) = NULL; 1593 + if (atomic_read(&(*per_cpu_ptr(d->sds, cpu))->ref)) 1594 + *per_cpu_ptr(d->sds, cpu) = NULL; 1605 1595 1606 - if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref)) 1607 - *per_cpu_ptr(sdd->sds, cpu) = NULL; 1596 + for (sd = *per_cpu_ptr(d->sd, cpu); sd; sd = sd->parent) { 1597 + struct sd_data *sdd = sd->private; 1608 1598 1609 - if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) 1610 - *per_cpu_ptr(sdd->sg, cpu) = NULL; 1599 + WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); 1600 + *per_cpu_ptr(sdd->sd, cpu) = NULL; 1611 1601 1612 - if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) 1613 - *per_cpu_ptr(sdd->sgc, cpu) = NULL; 1602 + if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref)) 1603 + *per_cpu_ptr(sdd->sds, cpu) = NULL; 1604 + 1605 + if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) 1606 + *per_cpu_ptr(sdd->sg, cpu) = NULL; 1607 + 1608 + if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) 1609 + *per_cpu_ptr(sdd->sgc, cpu) = NULL; 1610 + } 1614 1611 } 1615 1612 1616 1613 #ifdef CONFIG_NUMA ··· 1753 1736 #endif /* CONFIG_NUMA */ 1754 1737 } else { 1755 1738 sd->cache_nice_tries = 1; 1756 - } 1757 - 1758 - /* 1759 - * For all levels sharing cache; connect a sched_domain_shared 1760 - * instance. 1761 - */ 1762 - if (sd->flags & SD_SHARE_LLC) { 1763 - sd->shared = *per_cpu_ptr(sdd->sds, sd_id); 1764 - atomic_inc(&sd->shared->ref); 1765 - atomic_set(&sd->shared->nr_busy_cpus, sd_weight); 1766 1739 } 1767 1740 1768 1741 sd->private = sdd; ··· 2736 2729 while (sd->parent && (sd->parent->flags & SD_SHARE_LLC)) 2737 2730 sd = sd->parent; 2738 2731 2739 - /* 2740 - * In presence of higher domains, adjust the 2741 - * NUMA imbalance stats for the hierarchy. 2742 - */ 2743 - if (IS_ENABLED(CONFIG_NUMA) && (sd->flags & SD_SHARE_LLC) && sd->parent) 2744 - adjust_numa_imbalance(sd); 2732 + if (sd->flags & SD_SHARE_LLC) { 2733 + int sd_id = cpumask_first(sched_domain_span(sd)); 2734 + 2735 + sd->shared = *per_cpu_ptr(d.sds, sd_id); 2736 + atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); 2737 + atomic_inc(&sd->shared->ref); 2738 + 2739 + /* 2740 + * In presence of higher domains, adjust the 2741 + * NUMA imbalance stats for the hierarchy. 2742 + */ 2743 + if (IS_ENABLED(CONFIG_NUMA) && sd->parent) 2744 + adjust_numa_imbalance(sd); 2745 + } 2745 2746 } 2746 2747 2747 2748 /* Calculate CPU capacity for physical packages and nodes */ ··· 2757 2742 if (!cpumask_test_cpu(i, cpu_map)) 2758 2743 continue; 2759 2744 2760 - for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 2761 - claim_allocations(i, sd); 2745 + claim_allocations(i, &d); 2746 + 2747 + for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) 2762 2748 init_sched_groups_capacity(i, sd); 2763 - } 2764 2749 } 2765 2750 2766 2751 /* Attach the domains */

Configure Feed

Configure Feed