Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm: memcontrol: eliminate the problem of dying memory cgroup for LRU folios
Now that everything is set up, switch folio->memcg_data pointers to
objcgs, update the accessors, and execute reparenting on cgroup death.
Finally, folio->memcg_data of LRU folios and kmem folios will always point
to an object cgroup pointer. The folio->memcg_data of slab folios will
point to an vector of object cgroups.
Link: https://lore.kernel.org/80cb7af198dc6f2173fe616d1207a4c315ece141.1772711148.git.zhengqi.arch@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Allen Pais <apais@linux.microsoft.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chen Ridong <chenridong@huawei.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Hamza Mahfooz <hamzamahfooz@linux.microsoft.com>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Imran Khan <imran.f.khan@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kamalesh Babulal <kamalesh.babulal@oracle.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <Liam.Howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yosry Ahmed <yosry@kernel.org>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Muchun Song
and committed by
Andrew Morton
f1cf8d2f
01b9da29
+151
-135
+25
-52
include/linux/memcontrol.h
+25
-52
include/linux/memcontrol.h
···
369
369
#define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1)
370
370
371
371
#ifdef CONFIG_MEMCG
372
-
373
-
static inline bool folio_memcg_kmem(struct folio *folio);
374
-
375
372
/*
376
373
* After the initialization objcg->memcg is always pointing at
377
374
* a valid memcg, but can be atomically swapped to the parent memcg.
···
382
385
}
383
386
384
387
/*
385
-
* __folio_memcg - Get the memory cgroup associated with a non-kmem folio
386
-
* @folio: Pointer to the folio.
387
-
*
388
-
* Returns a pointer to the memory cgroup associated with the folio,
389
-
* or NULL. This function assumes that the folio is known to have a
390
-
* proper memory cgroup pointer. It's not safe to call this function
391
-
* against some type of folios, e.g. slab folios or ex-slab folios or
392
-
* kmem folios.
393
-
*/
394
-
static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
395
-
{
396
-
unsigned long memcg_data = folio->memcg_data;
397
-
398
-
VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
399
-
VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
400
-
VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);
401
-
402
-
return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
403
-
}
404
-
405
-
/*
406
-
* __folio_objcg - get the object cgroup associated with a kmem folio.
388
+
* folio_objcg - get the object cgroup associated with a folio.
407
389
* @folio: Pointer to the folio.
408
390
*
409
391
* Returns a pointer to the object cgroup associated with the folio,
410
392
* or NULL. This function assumes that the folio is known to have a
411
-
* proper object cgroup pointer. It's not safe to call this function
412
-
* against some type of folios, e.g. slab folios or ex-slab folios or
413
-
* LRU folios.
393
+
* proper object cgroup pointer.
414
394
*/
415
-
static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
395
+
static inline struct obj_cgroup *folio_objcg(struct folio *folio)
416
396
{
417
397
unsigned long memcg_data = folio->memcg_data;
418
398
419
399
VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
420
400
VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
421
-
VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);
422
401
423
402
return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
424
403
}
···
408
435
* proper memory cgroup pointer. It's not safe to call this function
409
436
* against some type of folios, e.g. slab folios or ex-slab folios.
410
437
*
411
-
* For a non-kmem folio any of the following ensures folio and memcg binding
412
-
* stability:
438
+
* For a folio any of the following ensures folio and objcg binding stability:
413
439
*
414
440
* - the folio lock
415
441
* - LRU isolation
416
442
* - exclusive reference
417
443
*
418
-
* For a kmem folio a caller should hold an rcu read lock to protect memcg
419
-
* associated with a kmem folio from being released.
444
+
* Based on the stable binding of folio and objcg, for a folio any of the
445
+
* following ensures folio and memcg binding stability:
446
+
*
447
+
* - cgroup_mutex
448
+
* - the lruvec lock
449
+
*
450
+
* If the caller only want to ensure that the page counters of memcg are
451
+
* updated correctly, ensure that the binding stability of folio and objcg
452
+
* is sufficient.
453
+
*
454
+
* Note: The caller should hold an rcu read lock or cgroup_mutex to protect
455
+
* memcg associated with a folio from being released.
420
456
*/
421
457
static inline struct mem_cgroup *folio_memcg(struct folio *folio)
422
458
{
423
-
if (folio_memcg_kmem(folio))
424
-
return obj_cgroup_memcg(__folio_objcg(folio));
425
-
return __folio_memcg(folio);
459
+
struct obj_cgroup *objcg = folio_objcg(folio);
460
+
461
+
return objcg ? obj_cgroup_memcg(objcg) : NULL;
426
462
}
427
463
428
464
/*
···
455
473
* has an associated memory cgroup pointer or an object cgroups vector or
456
474
* an object cgroup.
457
475
*
458
-
* For a non-kmem folio any of the following ensures folio and memcg binding
459
-
* stability:
476
+
* The page and objcg or memcg binding rules can refer to folio_memcg().
460
477
*
461
-
* - the folio lock
462
-
* - LRU isolation
463
-
* - exclusive reference
464
-
*
465
-
* For a kmem folio a caller should hold an rcu read lock to protect memcg
466
-
* associated with a kmem folio from being released.
478
+
* A caller should hold an rcu read lock to protect memcg associated with a
479
+
* page from being released.
467
480
*/
468
481
static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
469
482
{
···
467
490
* for slabs, READ_ONCE() should be used here.
468
491
*/
469
492
unsigned long memcg_data = READ_ONCE(folio->memcg_data);
493
+
struct obj_cgroup *objcg;
470
494
471
495
if (memcg_data & MEMCG_DATA_OBJEXTS)
472
496
return NULL;
473
497
474
-
if (memcg_data & MEMCG_DATA_KMEM) {
475
-
struct obj_cgroup *objcg;
498
+
objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
476
499
477
-
objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
478
-
return obj_cgroup_memcg(objcg);
479
-
}
480
-
481
-
return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
500
+
return objcg ? obj_cgroup_memcg(objcg) : NULL;
482
501
}
483
502
484
503
static inline struct mem_cgroup *page_memcg_check(struct page *page)
+9
-6
mm/memcontrol-v1.c
+9
-6
mm/memcontrol-v1.c
···
613
613
void memcg1_swapout(struct folio *folio, swp_entry_t entry)
614
614
{
615
615
struct mem_cgroup *memcg, *swap_memcg;
616
+
struct obj_cgroup *objcg;
616
617
unsigned int nr_entries;
617
618
618
619
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
···
625
624
if (!do_memsw_account())
626
625
return;
627
626
628
-
memcg = folio_memcg(folio);
629
-
630
-
VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
631
-
if (!memcg)
627
+
objcg = folio_objcg(folio);
628
+
VM_WARN_ON_ONCE_FOLIO(!objcg, folio);
629
+
if (!objcg)
632
630
return;
633
631
632
+
rcu_read_lock();
633
+
memcg = obj_cgroup_memcg(objcg);
634
634
/*
635
635
* In case the memcg owning these pages has been offlined and doesn't
636
636
* have an ID allocated to it anymore, charge the closest online
···
646
644
folio_unqueue_deferred_split(folio);
647
645
folio->memcg_data = 0;
648
646
649
-
if (!mem_cgroup_is_root(memcg))
647
+
if (!obj_cgroup_is_root(objcg))
650
648
page_counter_uncharge(&memcg->memory, nr_entries);
651
649
652
650
if (memcg != swap_memcg) {
···
667
665
preempt_enable_nested();
668
666
memcg1_check_events(memcg, folio_nid(folio));
669
667
670
-
css_put(&memcg->css);
668
+
rcu_read_unlock();
669
+
obj_cgroup_put(objcg);
671
670
}
672
671
673
672
/*
+117
-77
mm/memcontrol.c
+117
-77
mm/memcontrol.c
···
254
254
}
255
255
#endif
256
256
257
-
static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent)
257
+
static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
258
258
{
259
259
spin_lock_irq(&objcg_lock);
260
+
spin_lock_nested(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock, 1);
261
+
spin_lock_nested(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock, 2);
260
262
}
261
263
262
-
static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent)
264
+
static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
263
265
{
266
+
spin_unlock(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock);
267
+
spin_unlock(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock);
264
268
spin_unlock_irq(&objcg_lock);
265
269
}
266
270
···
275
271
int nid;
276
272
277
273
for_each_node(nid) {
278
-
reparent_locks(memcg, parent);
274
+
retry:
275
+
if (lru_gen_enabled())
276
+
max_lru_gen_memcg(parent, nid);
277
+
278
+
reparent_locks(memcg, parent, nid);
279
+
280
+
if (lru_gen_enabled()) {
281
+
if (!recheck_lru_gen_max_memcg(parent, nid)) {
282
+
reparent_unlocks(memcg, parent, nid);
283
+
cond_resched();
284
+
goto retry;
285
+
}
286
+
lru_gen_reparent_memcg(memcg, parent, nid);
287
+
} else {
288
+
lru_reparent_memcg(memcg, parent, nid);
289
+
}
279
290
280
291
objcg = __memcg_reparent_objcgs(memcg, parent, nid);
281
292
282
-
reparent_unlocks(memcg, parent);
293
+
reparent_unlocks(memcg, parent, nid);
283
294
284
295
percpu_ref_kill(&objcg->refcnt);
285
296
}
297
+
298
+
reparent_state_local(memcg, parent);
286
299
}
287
300
288
301
/*
···
844
823
this_cpu_add(memcg->vmstats_percpu->state[i], val);
845
824
val = memcg_state_val_in_pages(idx, val);
846
825
memcg_rstat_updated(memcg, val, cpu);
826
+
847
827
trace_mod_memcg_state(memcg, idx, val);
848
828
849
829
put_cpu();
···
862
840
if (mem_cgroup_disabled())
863
841
return;
864
842
843
+
memcg = get_non_dying_memcg_start(memcg);
865
844
__mod_memcg_state(memcg, idx, val);
845
+
get_non_dying_memcg_end();
866
846
}
867
847
868
848
#ifdef CONFIG_MEMCG_V1
···
924
900
enum node_stat_item idx,
925
901
int val)
926
902
{
903
+
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
927
904
struct mem_cgroup_per_node *pn;
905
+
struct mem_cgroup *memcg;
928
906
929
907
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
908
+
memcg = get_non_dying_memcg_start(pn->memcg);
909
+
pn = memcg->nodeinfo[pgdat->node_id];
930
910
931
911
__mod_memcg_lruvec_state(pn, idx, val);
912
+
913
+
get_non_dying_memcg_end();
932
914
}
933
915
934
916
/**
···
1157
1127
/**
1158
1128
* get_mem_cgroup_from_folio - Obtain a reference on a given folio's memcg.
1159
1129
* @folio: folio from which memcg should be extracted.
1130
+
*
1131
+
* See folio_memcg() for folio->objcg/memcg binding rules.
1160
1132
*/
1161
1133
struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)
1162
1134
{
···
2754
2722
return try_charge_memcg(memcg, gfp_mask, nr_pages);
2755
2723
}
2756
2724
2757
-
static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
2725
+
static void commit_charge(struct folio *folio, struct obj_cgroup *objcg)
2758
2726
{
2759
2727
VM_BUG_ON_FOLIO(folio_memcg_charged(folio), folio);
2760
2728
/*
2761
-
* Any of the following ensures page's memcg stability:
2729
+
* Any of the following ensures folio's objcg stability:
2762
2730
*
2763
2731
* - the page lock
2764
2732
* - LRU isolation
2765
2733
* - exclusive reference
2766
2734
*/
2767
-
folio->memcg_data = (unsigned long)memcg;
2735
+
folio->memcg_data = (unsigned long)objcg;
2768
2736
}
2769
2737
2770
2738
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
···
2878
2846
return NULL;
2879
2847
}
2880
2848
2849
+
static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
2850
+
{
2851
+
struct obj_cgroup *objcg;
2852
+
2853
+
rcu_read_lock();
2854
+
objcg = __get_obj_cgroup_from_memcg(memcg);
2855
+
rcu_read_unlock();
2856
+
2857
+
return objcg;
2858
+
}
2859
+
2881
2860
static struct obj_cgroup *current_objcg_update(void)
2882
2861
{
2883
2862
struct mem_cgroup *memcg;
···
2990
2947
{
2991
2948
struct obj_cgroup *objcg;
2992
2949
2993
-
if (!memcg_kmem_online())
2994
-
return NULL;
2995
-
2996
-
if (folio_memcg_kmem(folio)) {
2997
-
objcg = __folio_objcg(folio);
2950
+
objcg = folio_objcg(folio);
2951
+
if (objcg)
2998
2952
obj_cgroup_get(objcg);
2999
-
} else {
3000
-
rcu_read_lock();
3001
-
objcg = __get_obj_cgroup_from_memcg(__folio_memcg(folio));
3002
-
rcu_read_unlock();
3003
-
}
2953
+
3004
2954
return objcg;
3005
2955
}
3006
2956
···
3555
3519
return;
3556
3520
3557
3521
new_refs = (1 << (old_order - new_order)) - 1;
3558
-
css_get_many(&__folio_memcg(folio)->css, new_refs);
3522
+
obj_cgroup_get_many(folio_objcg(folio), new_refs);
3559
3523
}
3560
3524
3561
3525
static void memcg_online_kmem(struct mem_cgroup *memcg)
···
4991
4955
static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
4992
4956
gfp_t gfp)
4993
4957
{
4994
-
int ret;
4958
+
int ret = 0;
4959
+
struct obj_cgroup *objcg;
4995
4960
4996
-
ret = try_charge(memcg, gfp, folio_nr_pages(folio));
4997
-
if (ret)
4998
-
goto out;
4999
-
5000
-
css_get(&memcg->css);
5001
-
commit_charge(folio, memcg);
4961
+
objcg = get_obj_cgroup_from_memcg(memcg);
4962
+
/* Do not account at the root objcg level. */
4963
+
if (!obj_cgroup_is_root(objcg))
4964
+
ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio));
4965
+
if (ret) {
4966
+
obj_cgroup_put(objcg);
4967
+
return ret;
4968
+
}
4969
+
commit_charge(folio, objcg);
5002
4970
memcg1_commit_charge(folio, memcg);
5003
-
out:
4971
+
5004
4972
return ret;
5005
4973
}
5006
4974
···
5090
5050
}
5091
5051
5092
5052
struct uncharge_gather {
5093
-
struct mem_cgroup *memcg;
5053
+
struct obj_cgroup *objcg;
5094
5054
unsigned long nr_memory;
5095
5055
unsigned long pgpgout;
5096
5056
unsigned long nr_kmem;
···
5104
5064
5105
5065
static void uncharge_batch(const struct uncharge_gather *ug)
5106
5066
{
5067
+
struct mem_cgroup *memcg;
5068
+
5069
+
rcu_read_lock();
5070
+
memcg = obj_cgroup_memcg(ug->objcg);
5107
5071
if (ug->nr_memory) {
5108
-
memcg_uncharge(ug->memcg, ug->nr_memory);
5072
+
memcg_uncharge(memcg, ug->nr_memory);
5109
5073
if (ug->nr_kmem) {
5110
-
mod_memcg_state(ug->memcg, MEMCG_KMEM, -ug->nr_kmem);
5111
-
memcg1_account_kmem(ug->memcg, -ug->nr_kmem);
5074
+
mod_memcg_state(memcg, MEMCG_KMEM, -ug->nr_kmem);
5075
+
memcg1_account_kmem(memcg, -ug->nr_kmem);
5112
5076
}
5113
-
memcg1_oom_recover(ug->memcg);
5077
+
memcg1_oom_recover(memcg);
5114
5078
}
5115
5079
5116
-
memcg1_uncharge_batch(ug->memcg, ug->pgpgout, ug->nr_memory, ug->nid);
5080
+
memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory, ug->nid);
5081
+
rcu_read_unlock();
5117
5082
5118
5083
/* drop reference from uncharge_folio */
5119
-
css_put(&ug->memcg->css);
5084
+
obj_cgroup_put(ug->objcg);
5120
5085
}
5121
5086
5122
5087
static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
5123
5088
{
5124
5089
long nr_pages;
5125
-
struct mem_cgroup *memcg;
5126
5090
struct obj_cgroup *objcg;
5127
5091
5128
5092
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
5129
5093
5130
5094
/*
5131
5095
* Nobody should be changing or seriously looking at
5132
-
* folio memcg or objcg at this point, we have fully
5133
-
* exclusive access to the folio.
5096
+
* folio objcg at this point, we have fully exclusive
5097
+
* access to the folio.
5134
5098
*/
5135
-
if (folio_memcg_kmem(folio)) {
5136
-
objcg = __folio_objcg(folio);
5137
-
/*
5138
-
* This get matches the put at the end of the function and
5139
-
* kmem pages do not hold memcg references anymore.
5140
-
*/
5141
-
memcg = get_mem_cgroup_from_objcg(objcg);
5142
-
} else {
5143
-
memcg = __folio_memcg(folio);
5144
-
}
5145
-
5146
-
if (!memcg)
5099
+
objcg = folio_objcg(folio);
5100
+
if (!objcg)
5147
5101
return;
5148
5102
5149
-
if (ug->memcg != memcg) {
5150
-
if (ug->memcg) {
5103
+
if (ug->objcg != objcg) {
5104
+
if (ug->objcg) {
5151
5105
uncharge_batch(ug);
5152
5106
uncharge_gather_clear(ug);
5153
5107
}
5154
-
ug->memcg = memcg;
5108
+
ug->objcg = objcg;
5155
5109
ug->nid = folio_nid(folio);
5156
5110
5157
-
/* pairs with css_put in uncharge_batch */
5158
-
css_get(&memcg->css);
5111
+
/* pairs with obj_cgroup_put in uncharge_batch */
5112
+
obj_cgroup_get(objcg);
5159
5113
}
5160
5114
5161
5115
nr_pages = folio_nr_pages(folio);
···
5157
5123
if (folio_memcg_kmem(folio)) {
5158
5124
ug->nr_memory += nr_pages;
5159
5125
ug->nr_kmem += nr_pages;
5160
-
5161
-
folio->memcg_data = 0;
5162
-
obj_cgroup_put(objcg);
5163
5126
} else {
5164
5127
/* LRU pages aren't accounted at the root level */
5165
-
if (!mem_cgroup_is_root(memcg))
5128
+
if (!obj_cgroup_is_root(objcg))
5166
5129
ug->nr_memory += nr_pages;
5167
5130
ug->pgpgout++;
5168
5131
5169
5132
WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
5170
-
folio->memcg_data = 0;
5171
5133
}
5172
5134
5173
-
css_put(&memcg->css);
5135
+
folio->memcg_data = 0;
5136
+
obj_cgroup_put(objcg);
5174
5137
}
5175
5138
5176
5139
void __mem_cgroup_uncharge(struct folio *folio)
···
5191
5160
uncharge_gather_clear(&ug);
5192
5161
for (i = 0; i < folios->nr; i++)
5193
5162
uncharge_folio(folios->folios[i], &ug);
5194
-
if (ug.memcg)
5163
+
if (ug.objcg)
5195
5164
uncharge_batch(&ug);
5196
5165
}
5197
5166
···
5208
5177
void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
5209
5178
{
5210
5179
struct mem_cgroup *memcg;
5180
+
struct obj_cgroup *objcg;
5211
5181
long nr_pages = folio_nr_pages(new);
5212
5182
5213
5183
VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
···
5223
5191
if (folio_memcg_charged(new))
5224
5192
return;
5225
5193
5226
-
memcg = folio_memcg(old);
5227
-
VM_WARN_ON_ONCE_FOLIO(!memcg, old);
5228
-
if (!memcg)
5194
+
objcg = folio_objcg(old);
5195
+
VM_WARN_ON_ONCE_FOLIO(!objcg, old);
5196
+
if (!objcg)
5229
5197
return;
5230
5198
5199
+
rcu_read_lock();
5200
+
memcg = obj_cgroup_memcg(objcg);
5231
5201
/* Force-charge the new page. The old one will be freed soon */
5232
-
if (!mem_cgroup_is_root(memcg)) {
5202
+
if (!obj_cgroup_is_root(objcg)) {
5233
5203
page_counter_charge(&memcg->memory, nr_pages);
5234
5204
if (do_memsw_account())
5235
5205
page_counter_charge(&memcg->memsw, nr_pages);
5236
5206
}
5237
5207
5238
-
css_get(&memcg->css);
5239
-
commit_charge(new, memcg);
5208
+
obj_cgroup_get(objcg);
5209
+
commit_charge(new, objcg);
5240
5210
memcg1_commit_charge(new, memcg);
5211
+
rcu_read_unlock();
5241
5212
}
5242
5213
5243
5214
/**
···
5256
5221
*/
5257
5222
void mem_cgroup_migrate(struct folio *old, struct folio *new)
5258
5223
{
5259
-
struct mem_cgroup *memcg;
5224
+
struct obj_cgroup *objcg;
5260
5225
5261
5226
VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
5262
5227
VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
···
5267
5232
if (mem_cgroup_disabled())
5268
5233
return;
5269
5234
5270
-
memcg = folio_memcg(old);
5235
+
objcg = folio_objcg(old);
5271
5236
/*
5272
-
* Note that it is normal to see !memcg for a hugetlb folio.
5237
+
* Note that it is normal to see !objcg for a hugetlb folio.
5273
5238
* For e.g, it could have been allocated when memory_hugetlb_accounting
5274
5239
* was not selected.
5275
5240
*/
5276
-
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(old) && !memcg, old);
5277
-
if (!memcg)
5241
+
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(old) && !objcg, old);
5242
+
if (!objcg)
5278
5243
return;
5279
5244
5280
-
/* Transfer the charge and the css ref */
5281
-
commit_charge(new, memcg);
5245
+
/* Transfer the charge and the objcg ref */
5246
+
commit_charge(new, objcg);
5282
5247
5283
5248
/* Warning should never happen, so don't worry about refcount non-0 */
5284
5249
WARN_ON_ONCE(folio_unqueue_deferred_split(old));
···
5461
5426
unsigned int nr_pages = folio_nr_pages(folio);
5462
5427
struct page_counter *counter;
5463
5428
struct mem_cgroup *memcg;
5429
+
struct obj_cgroup *objcg;
5464
5430
5465
5431
if (do_memsw_account())
5466
5432
return 0;
5467
5433
5468
-
memcg = folio_memcg(folio);
5469
-
5470
-
VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
5471
-
if (!memcg)
5434
+
objcg = folio_objcg(folio);
5435
+
VM_WARN_ON_ONCE_FOLIO(!objcg, folio);
5436
+
if (!objcg)
5472
5437
return 0;
5473
5438
5439
+
rcu_read_lock();
5440
+
memcg = obj_cgroup_memcg(objcg);
5474
5441
if (!entry.val) {
5475
5442
memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
5443
+
rcu_read_unlock();
5476
5444
return 0;
5477
5445
}
5478
5446
5479
5447
memcg = mem_cgroup_private_id_get_online(memcg, nr_pages);
5448
+
/* memcg is pined by memcg ID. */
5449
+
rcu_read_unlock();
5480
5450
5481
5451
if (!mem_cgroup_is_root(memcg) &&
5482
5452
!page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {