Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2#include <fcntl.h>
3#include <limits.h>
4#include <stdio.h>
5#include <stdlib.h>
6#include <string.h>
7#include <unistd.h>
8
9#include "vm_util.h"
10#include "thp_settings.h"
11
12#define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
13#define MAX_SETTINGS_DEPTH 4
14static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH];
15static int settings_index;
16static struct thp_settings saved_settings;
17static char dev_queue_read_ahead_path[PATH_MAX];
18
19static const char * const thp_enabled_strings[] = {
20 "never",
21 "always",
22 "inherit",
23 "madvise",
24 NULL
25};
26
27static const char * const thp_defrag_strings[] = {
28 "always",
29 "defer",
30 "defer+madvise",
31 "madvise",
32 "never",
33 NULL
34};
35
36static const char * const shmem_enabled_strings[] = {
37 "never",
38 "always",
39 "within_size",
40 "advise",
41 "inherit",
42 "deny",
43 "force",
44 NULL
45};
46
47int read_file(const char *path, char *buf, size_t buflen)
48{
49 int fd;
50 ssize_t numread;
51
52 fd = open(path, O_RDONLY);
53 if (fd == -1)
54 return 0;
55
56 numread = read(fd, buf, buflen - 1);
57 if (numread < 1) {
58 close(fd);
59 return 0;
60 }
61
62 buf[numread] = '\0';
63 close(fd);
64
65 return (unsigned int) numread;
66}
67
68unsigned long read_num(const char *path)
69{
70 char buf[21];
71
72 if (read_file(path, buf, sizeof(buf)) < 0) {
73 perror("read_file()");
74 exit(EXIT_FAILURE);
75 }
76
77 return strtoul(buf, NULL, 10);
78}
79
80void write_num(const char *path, unsigned long num)
81{
82 char buf[21];
83
84 sprintf(buf, "%ld", num);
85 write_file(path, buf, strlen(buf) + 1);
86}
87
88int thp_read_string(const char *name, const char * const strings[])
89{
90 char path[PATH_MAX];
91 char buf[256];
92 char *c;
93 int ret;
94
95 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
96 if (ret >= PATH_MAX) {
97 printf("%s: Pathname is too long\n", __func__);
98 exit(EXIT_FAILURE);
99 }
100
101 if (!read_file(path, buf, sizeof(buf))) {
102 perror(path);
103 exit(EXIT_FAILURE);
104 }
105
106 c = strchr(buf, '[');
107 if (!c) {
108 printf("%s: Parse failure\n", __func__);
109 exit(EXIT_FAILURE);
110 }
111
112 c++;
113 memmove(buf, c, sizeof(buf) - (c - buf));
114
115 c = strchr(buf, ']');
116 if (!c) {
117 printf("%s: Parse failure\n", __func__);
118 exit(EXIT_FAILURE);
119 }
120 *c = '\0';
121
122 ret = 0;
123 while (strings[ret]) {
124 if (!strcmp(strings[ret], buf))
125 return ret;
126 ret++;
127 }
128
129 printf("Failed to parse %s\n", name);
130 exit(EXIT_FAILURE);
131}
132
133void thp_write_string(const char *name, const char *val)
134{
135 char path[PATH_MAX];
136 int ret;
137
138 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
139 if (ret >= PATH_MAX) {
140 printf("%s: Pathname is too long\n", __func__);
141 exit(EXIT_FAILURE);
142 }
143 write_file(path, val, strlen(val) + 1);
144}
145
146unsigned long thp_read_num(const char *name)
147{
148 char path[PATH_MAX];
149 int ret;
150
151 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
152 if (ret >= PATH_MAX) {
153 printf("%s: Pathname is too long\n", __func__);
154 exit(EXIT_FAILURE);
155 }
156 return read_num(path);
157}
158
159void thp_write_num(const char *name, unsigned long num)
160{
161 char path[PATH_MAX];
162 int ret;
163
164 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
165 if (ret >= PATH_MAX) {
166 printf("%s: Pathname is too long\n", __func__);
167 exit(EXIT_FAILURE);
168 }
169 write_num(path, num);
170}
171
172void thp_read_settings(struct thp_settings *settings)
173{
174 unsigned long orders = thp_supported_orders();
175 unsigned long shmem_orders = thp_shmem_supported_orders();
176 char path[PATH_MAX];
177 int i;
178
179 *settings = (struct thp_settings) {
180 .thp_enabled = thp_read_string("enabled", thp_enabled_strings),
181 .thp_defrag = thp_read_string("defrag", thp_defrag_strings),
182 .shmem_enabled =
183 thp_read_string("shmem_enabled", shmem_enabled_strings),
184 .use_zero_page = thp_read_num("use_zero_page"),
185 };
186 settings->khugepaged = (struct khugepaged_settings) {
187 .defrag = thp_read_num("khugepaged/defrag"),
188 .alloc_sleep_millisecs =
189 thp_read_num("khugepaged/alloc_sleep_millisecs"),
190 .scan_sleep_millisecs =
191 thp_read_num("khugepaged/scan_sleep_millisecs"),
192 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"),
193 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"),
194 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"),
195 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"),
196 };
197 if (dev_queue_read_ahead_path[0])
198 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path);
199
200 for (i = 0; i < NR_ORDERS; i++) {
201 if (!((1 << i) & orders)) {
202 settings->hugepages[i].enabled = THP_NEVER;
203 continue;
204 }
205 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
206 (getpagesize() >> 10) << i);
207 settings->hugepages[i].enabled =
208 thp_read_string(path, thp_enabled_strings);
209 }
210
211 for (i = 0; i < NR_ORDERS; i++) {
212 if (!((1 << i) & shmem_orders)) {
213 settings->shmem_hugepages[i].enabled = SHMEM_NEVER;
214 continue;
215 }
216 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",
217 (getpagesize() >> 10) << i);
218 settings->shmem_hugepages[i].enabled =
219 thp_read_string(path, shmem_enabled_strings);
220 }
221}
222
223void thp_write_settings(struct thp_settings *settings)
224{
225 struct khugepaged_settings *khugepaged = &settings->khugepaged;
226 unsigned long orders = thp_supported_orders();
227 unsigned long shmem_orders = thp_shmem_supported_orders();
228 char path[PATH_MAX];
229 int enabled;
230 int i;
231
232 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]);
233 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]);
234 thp_write_string("shmem_enabled",
235 shmem_enabled_strings[settings->shmem_enabled]);
236 thp_write_num("use_zero_page", settings->use_zero_page);
237
238 thp_write_num("khugepaged/defrag", khugepaged->defrag);
239 thp_write_num("khugepaged/alloc_sleep_millisecs",
240 khugepaged->alloc_sleep_millisecs);
241 thp_write_num("khugepaged/scan_sleep_millisecs",
242 khugepaged->scan_sleep_millisecs);
243 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);
244 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
245 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
246 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
247
248 if (dev_queue_read_ahead_path[0])
249 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb);
250
251 for (i = 0; i < NR_ORDERS; i++) {
252 if (!((1 << i) & orders))
253 continue;
254 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
255 (getpagesize() >> 10) << i);
256 enabled = settings->hugepages[i].enabled;
257 thp_write_string(path, thp_enabled_strings[enabled]);
258 }
259
260 for (i = 0; i < NR_ORDERS; i++) {
261 if (!((1 << i) & shmem_orders))
262 continue;
263 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",
264 (getpagesize() >> 10) << i);
265 enabled = settings->shmem_hugepages[i].enabled;
266 thp_write_string(path, shmem_enabled_strings[enabled]);
267 }
268}
269
270struct thp_settings *thp_current_settings(void)
271{
272 if (!settings_index) {
273 printf("Fail: No settings set");
274 exit(EXIT_FAILURE);
275 }
276 return settings_stack + settings_index - 1;
277}
278
279void thp_push_settings(struct thp_settings *settings)
280{
281 if (settings_index >= MAX_SETTINGS_DEPTH) {
282 printf("Fail: Settings stack exceeded");
283 exit(EXIT_FAILURE);
284 }
285 settings_stack[settings_index++] = *settings;
286 thp_write_settings(thp_current_settings());
287}
288
289void thp_pop_settings(void)
290{
291 if (settings_index <= 0) {
292 printf("Fail: Settings stack empty");
293 exit(EXIT_FAILURE);
294 }
295 --settings_index;
296 thp_write_settings(thp_current_settings());
297}
298
299void thp_restore_settings(void)
300{
301 thp_write_settings(&saved_settings);
302}
303
304void thp_save_settings(void)
305{
306 thp_read_settings(&saved_settings);
307}
308
309void thp_set_read_ahead_path(char *path)
310{
311 if (!path) {
312 dev_queue_read_ahead_path[0] = '\0';
313 return;
314 }
315
316 strncpy(dev_queue_read_ahead_path, path,
317 sizeof(dev_queue_read_ahead_path));
318 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0';
319}
320
321static unsigned long __thp_supported_orders(bool is_shmem)
322{
323 unsigned long orders = 0;
324 char path[PATH_MAX];
325 char buf[256];
326 int ret, i;
327 char anon_dir[] = "enabled";
328 char shmem_dir[] = "shmem_enabled";
329
330 for (i = 0; i < NR_ORDERS; i++) {
331 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s",
332 (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir);
333 if (ret >= PATH_MAX) {
334 printf("%s: Pathname is too long\n", __func__);
335 exit(EXIT_FAILURE);
336 }
337
338 ret = read_file(path, buf, sizeof(buf));
339 if (ret)
340 orders |= 1UL << i;
341 }
342
343 return orders;
344}
345
346unsigned long thp_supported_orders(void)
347{
348 return __thp_supported_orders(false);
349}
350
351unsigned long thp_shmem_supported_orders(void)
352{
353 return __thp_supported_orders(true);
354}
355
356bool thp_available(void)
357{
358 if (access(THP_SYSFS, F_OK) != 0)
359 return false;
360 return true;
361}
362
363bool thp_is_enabled(void)
364{
365 if (!thp_available())
366 return false;
367
368 int mode = thp_read_string("enabled", thp_enabled_strings);
369
370 /* THP is considered enabled if it's either "always" or "madvise" */
371 return mode == 1 || mode == 3;
372}