tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

selftests/resctrl: Adjust effective L3 cache size with SNC enabled

Sub-NUMA Cluster divides CPUs sharing an L3 cache into separate NUMA
nodes. Systems may support splitting into either two, three, four or six
nodes. When SNC mode is enabled the effective amount of L3 cache
available for allocation is divided by the number of nodes per L3.

It's possible to detect which SNC mode is active by comparing the number
of CPUs that share a cache with CPU0, with the number of CPUs on node0.

Detect SNC mode once and let other tests inherit that information.

Update CFLAGS after including lib.mk in the Makefile so that fallthrough
macro can be used.

To check if SNC detection is reliable one can check the
/sys/devices/system/cpu/offline file. If it's empty, it means all cores
are operational and the ratio should be calculated correctly. If it has
any contents, it means the detected SNC mode can't be trusted and should
be disabled.

Check if detection was not reliable due to offline cpus. If it was skip
running tests since the results couldn't be trusted.

Co-developed-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Maciej Wieczor-Retman <maciej.wieczor-retman@intel.com>
Reviewed-by: Reinette Chatre <reinette.chatre@intel.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>

authored by

Maciej Wieczor-Retman and committed by
Shuah Khan a1cd99e7 89ae6438

+119 -1
+1
tools/testing/selftests/resctrl/Makefile
··· 8 8 LOCAL_HDRS += $(wildcard *.h) 9 9 10 10 include ../lib.mk 11 + CFLAGS += -I$(top_srcdir)/tools/include 11 12 12 13 $(OUTPUT)/resctrl_tests: $(wildcard *.c)
+5
tools/testing/selftests/resctrl/resctrl.h
··· 11 11 #include <signal.h> 12 12 #include <dirent.h> 13 13 #include <stdbool.h> 14 + #include <ctype.h> 14 15 #include <sys/stat.h> 15 16 #include <sys/ioctl.h> 16 17 #include <sys/mount.h> ··· 22 21 #include <sys/eventfd.h> 23 22 #include <asm/unistd.h> 24 23 #include <linux/perf_event.h> 24 + #include <linux/compiler.h> 25 25 #include "../kselftest.h" 26 26 27 27 #define MB (1024 * 1024) ··· 158 156 */ 159 157 extern volatile int *value_sink; 160 158 159 + extern int snc_unreliable; 160 + 161 161 extern char llc_occup_path[1024]; 162 162 163 + int snc_nodes_per_l3_cache(void); 163 164 int get_vendor(void); 164 165 bool check_resctrlfs_support(void); 165 166 int filter_dmesg(void);
+8 -1
tools/testing/selftests/resctrl/resctrl_tests.c
··· 118 118 119 119 static void run_single_test(const struct resctrl_test *test, const struct user_params *uparams) 120 120 { 121 - int ret; 121 + int ret, snc_mode; 122 122 123 123 if (test->disabled) 124 124 return; ··· 128 128 return; 129 129 } 130 130 131 + snc_mode = snc_nodes_per_l3_cache(); 132 + 131 133 ksft_print_msg("Starting %s test ...\n", test->name); 134 + 135 + if (snc_mode == 1 && snc_unreliable && get_vendor() == ARCH_INTEL) { 136 + ksft_test_result_skip("SNC detection unreliable due to offline CPUs. Test results may not be accurate if SNC enabled.\n"); 137 + return; 138 + } 132 139 133 140 if (test_prepare(test)) { 134 141 ksft_exit_fail_msg("Abnormal failure when preparing for the test\n");
+105
tools/testing/selftests/resctrl/resctrlfs.c
··· 13 13 14 14 #include "resctrl.h" 15 15 16 + int snc_unreliable; 17 + 16 18 static int find_resctrl_mount(char *buffer) 17 19 { 18 20 FILE *mounts; ··· 159 157 } 160 158 161 159 /* 160 + * Count number of CPUs in a /sys bitmap 161 + */ 162 + static unsigned int count_sys_bitmap_bits(char *name) 163 + { 164 + FILE *fp = fopen(name, "r"); 165 + int count = 0, c; 166 + 167 + if (!fp) 168 + return 0; 169 + 170 + while ((c = fgetc(fp)) != EOF) { 171 + if (!isxdigit(c)) 172 + continue; 173 + switch (c) { 174 + case 'f': 175 + count++; 176 + fallthrough; 177 + case '7': case 'b': case 'd': case 'e': 178 + count++; 179 + fallthrough; 180 + case '3': case '5': case '6': case '9': case 'a': case 'c': 181 + count++; 182 + fallthrough; 183 + case '1': case '2': case '4': case '8': 184 + count++; 185 + break; 186 + } 187 + } 188 + fclose(fp); 189 + 190 + return count; 191 + } 192 + 193 + static bool cpus_offline_empty(void) 194 + { 195 + char offline_cpus_str[64]; 196 + FILE *fp; 197 + 198 + fp = fopen("/sys/devices/system/cpu/offline", "r"); 199 + if (!fp) { 200 + ksft_perror("Could not open /sys/devices/system/cpu/offline"); 201 + return 0; 202 + } 203 + 204 + if (fscanf(fp, "%63s", offline_cpus_str) < 0) { 205 + if (!errno) { 206 + fclose(fp); 207 + return 1; 208 + } 209 + ksft_perror("Could not read /sys/devices/system/cpu/offline"); 210 + } 211 + 212 + fclose(fp); 213 + 214 + return 0; 215 + } 216 + 217 + /* 218 + * Detect SNC by comparing #CPUs in node0 with #CPUs sharing LLC with CPU0. 219 + * If any CPUs are offline declare the detection as unreliable. 220 + */ 221 + int snc_nodes_per_l3_cache(void) 222 + { 223 + int node_cpus, cache_cpus; 224 + static int snc_mode; 225 + 226 + if (!snc_mode) { 227 + snc_mode = 1; 228 + if (!cpus_offline_empty()) { 229 + ksft_print_msg("Runtime SNC detection unreliable due to offline CPUs.\n"); 230 + ksft_print_msg("Setting SNC mode to disabled.\n"); 231 + snc_unreliable = 1; 232 + return snc_mode; 233 + } 234 + node_cpus = count_sys_bitmap_bits("/sys/devices/system/node/node0/cpumap"); 235 + cache_cpus = count_sys_bitmap_bits("/sys/devices/system/cpu/cpu0/cache/index3/shared_cpu_map"); 236 + 237 + if (!node_cpus || !cache_cpus) { 238 + ksft_print_msg("Could not determine Sub-NUMA Cluster mode.\n"); 239 + snc_unreliable = 1; 240 + return snc_mode; 241 + } 242 + snc_mode = cache_cpus / node_cpus; 243 + 244 + if (snc_mode > 1) 245 + ksft_print_msg("SNC-%d mode discovered.\n", snc_mode); 246 + } 247 + 248 + return snc_mode; 249 + } 250 + 251 + /* 162 252 * get_cache_size - Get cache size for a specified CPU 163 253 * @cpu_no: CPU number 164 254 * @cache_type: Cache level L2/L3 ··· 305 211 break; 306 212 } 307 213 214 + /* 215 + * The amount of cache represented by each bit in the masks 216 + * in the schemata file is reduced by a factor equal to SNC 217 + * nodes per L3 cache. 218 + * E.g. on a SNC-2 system with a 100MB L3 cache a test that 219 + * allocates memory from its local SNC node (default behavior 220 + * without using libnuma) will only see 50 MB llc_occupancy 221 + * with a fully populated L3 mask in the schemata file. 222 + */ 223 + if (cache_num == 3) 224 + *cache_size /= snc_nodes_per_l3_cache(); 308 225 return 0; 309 226 } 310 227