arm_mpam: Allow configuration to be applied and restored during cpu online

+291 -26

2 changed files

expand all

drivers

resctrl

mpam_devices.c

mpam_internal.h

+264 -26

drivers/resctrl/mpam_devices.c

··· 145 145 } 146 146 } 147 147 148 + /* 149 + * Once mpam is enabled, new requestors cannot further reduce the available 150 + * partid. Assert that the size is fixed, and new requestors will be turned 151 + * away. 152 + */ 153 + static void mpam_assert_partid_sizes_fixed(void) 154 + { 155 + WARN_ON_ONCE(!partid_max_published); 156 + } 157 + 148 158 static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg) 149 159 { 150 160 WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); ··· 348 338 return comp; 349 339 } 350 340 341 + static void __destroy_component_cfg(struct mpam_component *comp); 342 + 351 343 static void mpam_component_destroy(struct mpam_component *comp) 352 344 { 353 345 struct mpam_class *class = comp->class; 354 346 355 347 lockdep_assert_held(&mpam_list_lock); 348 + 349 + __destroy_component_cfg(comp); 356 350 357 351 list_del_rcu(&comp->class_list); 358 352 add_to_garbage(comp); ··· 833 819 __mpam_write_reg(msc, reg, bm); 834 820 } 835 821 836 - static void mpam_reset_ris_partid(struct mpam_msc_ris *ris, u16 partid) 822 + /* Called via IPI. Call while holding an SRCU reference */ 823 + static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, 824 + struct mpam_config *cfg) 837 825 { 838 826 struct mpam_msc *msc = ris->vmsc->msc; 839 827 struct mpam_props *rprops = &ris->props; 840 828 841 - WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu))); 842 - 843 829 mutex_lock(&msc->part_sel_lock); 844 830 __mpam_part_sel(ris->ris_idx, partid, msc); 845 831 846 - if (mpam_has_feature(mpam_feat_cpor_part, rprops)) 847 - mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); 832 + if (mpam_has_feature(mpam_feat_cpor_part, rprops) && 833 + mpam_has_feature(mpam_feat_cpor_part, cfg)) { 834 + if (cfg->reset_cpbm) 835 + mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); 836 + else 837 + mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); 838 + } 848 839 849 - if (mpam_has_feature(mpam_feat_mbw_part, rprops)) 850 - mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); 840 + if (mpam_has_feature(mpam_feat_mbw_part, rprops) && 841 + mpam_has_feature(mpam_feat_mbw_part, cfg)) { 842 + if (cfg->reset_mbw_pbm) 843 + mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); 844 + else 845 + mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); 846 + } 851 847 852 - if (mpam_has_feature(mpam_feat_mbw_min, rprops)) 848 + if (mpam_has_feature(mpam_feat_mbw_min, rprops) && 849 + mpam_has_feature(mpam_feat_mbw_min, cfg)) 853 850 mpam_write_partsel_reg(msc, MBW_MIN, 0); 854 851 855 - if (mpam_has_feature(mpam_feat_mbw_max, rprops)) 856 - mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); 852 + if (mpam_has_feature(mpam_feat_mbw_max, rprops) && 853 + mpam_has_feature(mpam_feat_mbw_max, cfg)) { 854 + if (cfg->reset_mbw_max) 855 + mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); 856 + else 857 + mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); 858 + } 857 859 858 860 mutex_unlock(&msc->part_sel_lock); 861 + } 862 + 863 + static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) 864 + { 865 + *reset_cfg = (struct mpam_config) { 866 + .reset_cpbm = true, 867 + .reset_mbw_pbm = true, 868 + .reset_mbw_max = true, 869 + }; 870 + bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST); 859 871 } 860 872 861 873 /* ··· 891 851 static int mpam_reset_ris(void *arg) 892 852 { 893 853 u16 partid, partid_max; 854 + struct mpam_config reset_cfg; 894 855 struct mpam_msc_ris *ris = arg; 895 856 896 857 if (ris->in_reset_state) 897 858 return 0; 898 859 860 + mpam_init_reset_cfg(&reset_cfg); 861 + 899 862 spin_lock(&partid_max_lock); 900 863 partid_max = mpam_partid_max; 901 864 spin_unlock(&partid_max_lock); 902 865 for (partid = 0; partid <= partid_max; partid++) 903 - mpam_reset_ris_partid(ris, partid); 866 + mpam_reprogram_ris_partid(ris, partid, &reset_cfg); 904 867 905 868 return 0; 906 869 } ··· 932 889 return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true); 933 890 } 934 891 935 - static void mpam_reset_msc(struct mpam_msc *msc, bool online) 936 - { 892 + struct mpam_write_config_arg { 937 893 struct mpam_msc_ris *ris; 894 + struct mpam_component *comp; 895 + u16 partid; 896 + }; 938 897 939 - list_for_each_entry_srcu(ris, &msc->ris, msc_list, srcu_read_lock_held(&mpam_srcu)) { 940 - mpam_touch_msc(msc, &mpam_reset_ris, ris); 898 + static int __write_config(void *arg) 899 + { 900 + struct mpam_write_config_arg *c = arg; 941 901 942 - /* 943 - * Set in_reset_state when coming online. The reset state 944 - * for non-zero partid may be lost while the CPUs are offline. 945 - */ 946 - ris->in_reset_state = online; 902 + mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]); 903 + 904 + return 0; 905 + } 906 + 907 + static void mpam_reprogram_msc(struct mpam_msc *msc) 908 + { 909 + u16 partid; 910 + bool reset; 911 + struct mpam_config *cfg; 912 + struct mpam_msc_ris *ris; 913 + struct mpam_write_config_arg arg; 914 + 915 + /* 916 + * No lock for mpam_partid_max as partid_max_published has been 917 + * set by mpam_enabled(), so the values can no longer change. 918 + */ 919 + mpam_assert_partid_sizes_fixed(); 920 + 921 + mutex_lock(&msc->cfg_lock); 922 + list_for_each_entry_srcu(ris, &msc->ris, msc_list, 923 + srcu_read_lock_held(&mpam_srcu)) { 924 + if (!mpam_is_enabled() && !ris->in_reset_state) { 925 + mpam_touch_msc(msc, &mpam_reset_ris, ris); 926 + ris->in_reset_state = true; 927 + continue; 928 + } 929 + 930 + arg.comp = ris->vmsc->comp; 931 + arg.ris = ris; 932 + reset = true; 933 + for (partid = 0; partid <= mpam_partid_max; partid++) { 934 + cfg = &ris->vmsc->comp->cfg[partid]; 935 + if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST)) 936 + reset = false; 937 + 938 + arg.partid = partid; 939 + mpam_touch_msc(msc, __write_config, &arg); 940 + } 941 + ris->in_reset_state = reset; 947 942 } 943 + mutex_unlock(&msc->cfg_lock); 948 944 } 949 945 950 946 static void _enable_percpu_irq(void *_irq) ··· 1007 925 _enable_percpu_irq(&msc->reenable_error_ppi); 1008 926 1009 927 if (atomic_fetch_inc(&msc->online_refs) == 0) 1010 - mpam_reset_msc(msc, true); 928 + mpam_reprogram_msc(msc); 1011 929 } 1012 930 1013 931 return 0; ··· 1062 980 if (msc->reenable_error_ppi) 1063 981 disable_percpu_irq(msc->reenable_error_ppi); 1064 982 1065 - if (atomic_dec_and_test(&msc->online_refs)) 1066 - mpam_reset_msc(msc, false); 983 + if (atomic_dec_and_test(&msc->online_refs)) { 984 + struct mpam_msc_ris *ris; 985 + 986 + mutex_lock(&msc->cfg_lock); 987 + list_for_each_entry_srcu(ris, &msc->ris, msc_list, 988 + srcu_read_lock_held(&mpam_srcu)) { 989 + mpam_touch_msc(msc, &mpam_reset_ris, ris); 990 + 991 + /* 992 + * The reset state for non-zero partid may be 993 + * lost while the CPUs are offline. 994 + */ 995 + ris->in_reset_state = false; 996 + } 997 + mutex_unlock(&msc->cfg_lock); 998 + } 1067 999 } 1068 1000 1069 1001 return 0; ··· 1217 1121 err = devm_mutex_init(dev, &msc->error_irq_lock); 1218 1122 if (err) 1219 1123 return ERR_PTR(err); 1124 + 1125 + err = devm_mutex_init(dev, &msc->cfg_lock); 1126 + if (err) 1127 + return ERR_PTR(err); 1128 + 1220 1129 mpam_mon_sel_lock_init(msc); 1221 1130 msc->id = pdev->id; 1222 1131 msc->pdev = pdev; ··· 1682 1581 } 1683 1582 } 1684 1583 1584 + static void __destroy_component_cfg(struct mpam_component *comp) 1585 + { 1586 + add_to_garbage(comp->cfg); 1587 + } 1588 + 1589 + static void mpam_reset_component_cfg(struct mpam_component *comp) 1590 + { 1591 + int i; 1592 + struct mpam_props *cprops = &comp->class->props; 1593 + 1594 + mpam_assert_partid_sizes_fixed(); 1595 + 1596 + if (!comp->cfg) 1597 + return; 1598 + 1599 + for (i = 0; i <= mpam_partid_max; i++) { 1600 + comp->cfg[i] = (struct mpam_config) {}; 1601 + if (cprops->cpbm_wd) 1602 + comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0); 1603 + if (cprops->mbw_pbm_bits) 1604 + comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); 1605 + if (cprops->bwa_wd) 1606 + comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); 1607 + } 1608 + } 1609 + 1610 + static int __allocate_component_cfg(struct mpam_component *comp) 1611 + { 1612 + mpam_assert_partid_sizes_fixed(); 1613 + 1614 + if (comp->cfg) 1615 + return 0; 1616 + 1617 + comp->cfg = kcalloc(mpam_partid_max + 1, sizeof(*comp->cfg), GFP_KERNEL); 1618 + if (!comp->cfg) 1619 + return -ENOMEM; 1620 + 1621 + /* 1622 + * The array is free()d in one go, so only cfg[0]'s structure needs 1623 + * to be initialised. 1624 + */ 1625 + init_garbage(&comp->cfg[0].garbage); 1626 + 1627 + mpam_reset_component_cfg(comp); 1628 + 1629 + return 0; 1630 + } 1631 + 1632 + static int mpam_allocate_config(void) 1633 + { 1634 + struct mpam_class *class; 1635 + struct mpam_component *comp; 1636 + 1637 + lockdep_assert_held(&mpam_list_lock); 1638 + 1639 + list_for_each_entry(class, &mpam_classes, classes_list) { 1640 + list_for_each_entry(comp, &class->components, class_list) { 1641 + int err = __allocate_component_cfg(comp); 1642 + if (err) 1643 + return err; 1644 + } 1645 + } 1646 + 1647 + return 0; 1648 + } 1649 + 1685 1650 static void mpam_enable_once(void) 1686 1651 { 1687 1652 int err; ··· 1767 1600 */ 1768 1601 cpus_read_lock(); 1769 1602 mutex_lock(&mpam_list_lock); 1770 - mpam_enable_merge_features(&mpam_classes); 1603 + do { 1604 + mpam_enable_merge_features(&mpam_classes); 1771 1605 1772 - err = mpam_register_irqs(); 1606 + err = mpam_register_irqs(); 1607 + if (err) { 1608 + pr_warn("Failed to register irqs: %d\n", err); 1609 + break; 1610 + } 1773 1611 1612 + err = mpam_allocate_config(); 1613 + if (err) { 1614 + pr_err("Failed to allocate configuration arrays.\n"); 1615 + break; 1616 + } 1617 + } while (0); 1774 1618 mutex_unlock(&mpam_list_lock); 1775 1619 cpus_read_unlock(); 1776 1620 1777 1621 if (err) { 1778 - pr_warn("Failed to register irqs: %d\n", err); 1779 1622 mpam_disable_reason = "Failed to enable."; 1780 1623 schedule_work(&mpam_broken_work); 1781 1624 return; ··· 1805 1628 struct mpam_vmsc *vmsc; 1806 1629 1807 1630 lockdep_assert_cpus_held(); 1631 + mpam_assert_partid_sizes_fixed(); 1632 + 1633 + mpam_reset_component_cfg(comp); 1808 1634 1809 1635 guard(srcu)(&mpam_srcu); 1810 1636 list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, ··· 1906 1726 1907 1727 if (all_devices_probed && !atomic_fetch_inc(&once)) 1908 1728 mpam_enable_once(); 1729 + } 1730 + 1731 + #define maybe_update_config(cfg, feature, newcfg, member, changes) do { \ 1732 + if (mpam_has_feature(feature, newcfg) && \ 1733 + (newcfg)->member != (cfg)->member) { \ 1734 + (cfg)->member = (newcfg)->member; \ 1735 + mpam_set_feature(feature, cfg); \ 1736 + \ 1737 + (changes) = true; \ 1738 + } \ 1739 + } while (0) 1740 + 1741 + static bool mpam_update_config(struct mpam_config *cfg, 1742 + const struct mpam_config *newcfg) 1743 + { 1744 + bool has_changes = false; 1745 + 1746 + maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes); 1747 + maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes); 1748 + maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes); 1749 + 1750 + return has_changes; 1751 + } 1752 + 1753 + int mpam_apply_config(struct mpam_component *comp, u16 partid, 1754 + struct mpam_config *cfg) 1755 + { 1756 + struct mpam_write_config_arg arg; 1757 + struct mpam_msc_ris *ris; 1758 + struct mpam_vmsc *vmsc; 1759 + struct mpam_msc *msc; 1760 + 1761 + lockdep_assert_cpus_held(); 1762 + 1763 + /* Don't pass in the current config! */ 1764 + WARN_ON_ONCE(&comp->cfg[partid] == cfg); 1765 + 1766 + if (!mpam_update_config(&comp->cfg[partid], cfg)) 1767 + return 0; 1768 + 1769 + arg.comp = comp; 1770 + arg.partid = partid; 1771 + 1772 + guard(srcu)(&mpam_srcu); 1773 + list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, 1774 + srcu_read_lock_held(&mpam_srcu)) { 1775 + msc = vmsc->msc; 1776 + 1777 + mutex_lock(&msc->cfg_lock); 1778 + list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, 1779 + srcu_read_lock_held(&mpam_srcu)) { 1780 + arg.ris = ris; 1781 + mpam_touch_msc(msc, __write_config, &arg); 1782 + } 1783 + mutex_unlock(&msc->cfg_lock); 1784 + } 1785 + 1786 + return 0; 1909 1787 } 1910 1788 1911 1789 static int __init mpam_msc_driver_init(void)

+27

drivers/resctrl/mpam_internal.h

··· 91 91 */ 92 92 struct mutex part_sel_lock; 93 93 94 + /* cfg_lock protects the msc configuration. */ 95 + struct mutex cfg_lock; 96 + 94 97 /* 95 98 * mon_sel_lock protects access to the MSC hardware registers that are 96 99 * affected by MPAMCFG_MON_SEL, and the mbwu_state. ··· 185 182 struct mpam_garbage garbage; 186 183 }; 187 184 185 + struct mpam_config { 186 + /* Which configuration values are valid. */ 187 + DECLARE_BITMAP(features, MPAM_FEATURE_LAST); 188 + 189 + u32 cpbm; 190 + u32 mbw_pbm; 191 + u16 mbw_max; 192 + 193 + bool reset_cpbm; 194 + bool reset_mbw_pbm; 195 + bool reset_mbw_max; 196 + 197 + struct mpam_garbage garbage; 198 + }; 199 + 188 200 struct mpam_component { 189 201 u32 comp_id; 190 202 ··· 207 189 struct list_head vmsc; 208 190 209 191 cpumask_t affinity; 192 + 193 + /* 194 + * Array of configuration values, indexed by partid. 195 + * Read from cpuhp callbacks, hold the cpuhp lock when writing. 196 + */ 197 + struct mpam_config *cfg; 210 198 211 199 /* member of mpam_class:components */ 212 200 struct list_head class_list; ··· 272 248 /* Scheduled work callback to enable mpam once all MSC have been probed */ 273 249 void mpam_enable(struct work_struct *work); 274 250 void mpam_disable(struct work_struct *work); 251 + 252 + int mpam_apply_config(struct mpam_component *comp, u16 partid, 253 + struct mpam_config *cfg); 275 254 276 255 int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, 277 256 cpumask_t *affinity);

Configure Feed

Configure Feed