Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
ttm/pool: track allocated_pages per numa node.
This gets the memory sizes from the nodes and stores the limit
as 50% of those. I think eventually we should drop the limits
once we have memcg aware shrinking, but this should be more NUMA
friendly, and I think seems like what people would prefer to
happen on NUMA aware systems.
Cc: Christian Koenig <christian.koenig@amd.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
+47
-15
+47
-15
drivers/gpu/drm/ttm/ttm_pool.c
+47
-15
drivers/gpu/drm/ttm/ttm_pool.c
···
116
116
117
117
static unsigned long page_pool_size;
118
118
119
-
MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");
119
+
MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool per NUMA node");
120
120
module_param(page_pool_size, ulong, 0644);
121
121
122
-
static atomic_long_t allocated_pages;
122
+
static unsigned long pool_node_limit[MAX_NUMNODES];
123
+
static atomic_long_t allocated_pages[MAX_NUMNODES];
123
124
124
125
static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS];
125
126
static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS];
···
300
299
static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p)
301
300
{
302
301
unsigned int i, num_pages = 1 << pt->order;
302
+
int nid = page_to_nid(p);
303
303
304
304
for (i = 0; i < num_pages; ++i) {
305
305
if (PageHighMem(p))
···
311
309
312
310
INIT_LIST_HEAD(&p->lru);
313
311
rcu_read_lock();
314
-
list_lru_add(&pt->pages, &p->lru, page_to_nid(p), NULL);
312
+
list_lru_add(&pt->pages, &p->lru, nid, NULL);
315
313
rcu_read_unlock();
316
-
atomic_long_add(1 << pt->order, &allocated_pages);
317
314
315
+
atomic_long_add(num_pages, &allocated_pages[nid]);
318
316
mod_lruvec_page_state(p, NR_GPU_ACTIVE, -num_pages);
319
317
mod_lruvec_page_state(p, NR_GPU_RECLAIM, num_pages);
320
318
}
···
340
338
341
339
ret = list_lru_walk_node(&pt->pages, nid, take_one_from_lru, (void *)&p, &nr_to_walk);
342
340
if (ret == 1 && p) {
343
-
atomic_long_sub(1 << pt->order, &allocated_pages);
341
+
atomic_long_sub(1 << pt->order, &allocated_pages[nid]);
344
342
mod_lruvec_page_state(p, NR_GPU_ACTIVE, (1 << pt->order));
345
343
mod_lruvec_page_state(p, NR_GPU_RECLAIM, -(1 << pt->order));
346
344
}
···
379
377
struct page *p;
380
378
p = list_first_entry(dispose, struct page, lru);
381
379
list_del_init(&p->lru);
382
-
atomic_long_sub(1 << pt->order, &allocated_pages);
380
+
atomic_long_sub(1 << pt->order, &allocated_pages[page_to_nid(p)]);
383
381
ttm_pool_free_page(pt->pool, pt->caching, pt->order, p, true);
384
382
}
385
383
}
···
942
940
*/
943
941
void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)
944
942
{
943
+
int nid = ttm_pool_nid(pool);
944
+
945
945
ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);
946
946
947
-
while (atomic_long_read(&allocated_pages) > page_pool_size) {
948
-
unsigned long diff = atomic_long_read(&allocated_pages) - page_pool_size;
949
-
ttm_pool_shrink(ttm_pool_nid(pool), diff);
947
+
while (atomic_long_read(&allocated_pages[nid]) > pool_node_limit[nid]) {
948
+
unsigned long diff = atomic_long_read(&allocated_pages[nid]) - pool_node_limit[nid];
949
+
ttm_pool_shrink(nid, diff);
950
950
}
951
951
}
952
952
EXPORT_SYMBOL(ttm_pool_free);
···
1206
1202
do
1207
1203
num_freed += ttm_pool_shrink(sc->nid, sc->nr_to_scan);
1208
1204
while (num_freed < sc->nr_to_scan &&
1209
-
atomic_long_read(&allocated_pages));
1205
+
atomic_long_read(&allocated_pages[sc->nid]));
1210
1206
1211
1207
sc->nr_scanned = num_freed;
1212
1208
···
1217
1213
static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink,
1218
1214
struct shrink_control *sc)
1219
1215
{
1220
-
unsigned long num_pages = atomic_long_read(&allocated_pages);
1216
+
unsigned long num_pages = atomic_long_read(&allocated_pages[sc->nid]);
1221
1217
1222
1218
return num_pages ? num_pages : SHRINK_EMPTY;
1223
1219
}
···
1254
1250
/* Dump the total amount of allocated pages */
1255
1251
static void ttm_pool_debugfs_footer(struct seq_file *m)
1256
1252
{
1257
-
seq_printf(m, "\ntotal\t: %8lu of %8lu\n",
1258
-
atomic_long_read(&allocated_pages), page_pool_size);
1253
+
int nid;
1254
+
1255
+
for_each_node(nid) {
1256
+
seq_printf(m, "\ntotal node%d\t: %8lu of %8lu\n", nid,
1257
+
atomic_long_read(&allocated_pages[nid]), pool_node_limit[nid]);
1258
+
}
1259
1259
}
1260
1260
1261
1261
/* Dump the information for the global pools */
···
1353
1345
1354
1346
#endif
1355
1347
1348
+
static inline u64 ttm_get_node_memory_size(int nid)
1349
+
{
1350
+
/*
1351
+
* This is directly using si_meminfo_node implementation as the
1352
+
* function is not exported.
1353
+
*/
1354
+
int zone_type;
1355
+
u64 managed_pages = 0;
1356
+
1357
+
pg_data_t *pgdat = NODE_DATA(nid);
1358
+
1359
+
for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
1360
+
managed_pages +=
1361
+
zone_managed_pages(&pgdat->node_zones[zone_type]);
1362
+
return managed_pages * PAGE_SIZE;
1363
+
}
1364
+
1356
1365
/**
1357
1366
* ttm_pool_mgr_init - Initialize globals
1358
1367
*
···
1381
1356
{
1382
1357
unsigned int i;
1383
1358
1384
-
if (!page_pool_size)
1385
-
page_pool_size = num_pages;
1359
+
int nid;
1360
+
for_each_node(nid) {
1361
+
if (!page_pool_size) {
1362
+
u64 node_size = ttm_get_node_memory_size(nid);
1363
+
pool_node_limit[nid] = (node_size >> PAGE_SHIFT) / 2;
1364
+
} else {
1365
+
pool_node_limit[nid] = page_pool_size;
1366
+
}
1367
+
}
1386
1368
1387
1369
spin_lock_init(&shrinker_lock);
1388
1370
INIT_LIST_HEAD(&shrinker_list);