Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'block-6.4-2023-06-15' of git://git.kernel.dk/linux
Pull block fix from Jens Axboe:
"Just a single fix for blk-cg stats flushing"
* tag 'block-6.4-2023-06-15' of git://git.kernel.dk/linux:
blk-cgroup: Flush stats before releasing blkcg_gq
+31
-9
+31
-9
block/blk-cgroup.c
+31
-9
block/blk-cgroup.c
···
34
34
#include "blk-ioprio.h"
35
35
#include "blk-throttle.h"
36
36
37
+
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
38
+
37
39
/*
38
40
* blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
39
41
* blkcg_pol_register_mutex nests outside of it and synchronizes entire
···
57
55
static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
58
56
59
57
bool blkcg_debug_stats = false;
58
+
59
+
static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
60
60
61
61
#define BLKG_DESTROY_BATCH_SIZE 64
62
62
···
167
163
static void __blkg_release(struct rcu_head *rcu)
168
164
{
169
165
struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
166
+
struct blkcg *blkcg = blkg->blkcg;
167
+
int cpu;
170
168
171
169
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
172
170
WARN_ON(!bio_list_empty(&blkg->async_bios));
173
171
#endif
172
+
/*
173
+
* Flush all the non-empty percpu lockless lists before releasing
174
+
* us, given these stat belongs to us.
175
+
*
176
+
* blkg_stat_lock is for serializing blkg stat update
177
+
*/
178
+
for_each_possible_cpu(cpu)
179
+
__blkcg_rstat_flush(blkcg, cpu);
174
180
175
181
/* release the blkcg and parent blkg refs this blkg has been holding */
176
182
css_put(&blkg->blkcg->css);
···
965
951
u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
966
952
}
967
953
968
-
static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
954
+
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
969
955
{
970
-
struct blkcg *blkcg = css_to_blkcg(css);
971
956
struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
972
957
struct llist_node *lnode;
973
958
struct blkg_iostat_set *bisc, *next_bisc;
974
-
975
-
/* Root-level stats are sourced from system-wide IO stats */
976
-
if (!cgroup_parent(css->cgroup))
977
-
return;
978
959
979
960
rcu_read_lock();
980
961
981
962
lnode = llist_del_all(lhead);
982
963
if (!lnode)
983
964
goto out;
965
+
966
+
/*
967
+
* For covering concurrent parent blkg update from blkg_release().
968
+
*
969
+
* When flushing from cgroup, cgroup_rstat_lock is always held, so
970
+
* this lock won't cause contention most of time.
971
+
*/
972
+
raw_spin_lock(&blkg_stat_lock);
984
973
985
974
/*
986
975
* Iterate only the iostat_cpu's queued in the lockless list.
···
1008
991
if (parent && parent->parent)
1009
992
blkcg_iostat_update(parent, &blkg->iostat.cur,
1010
993
&blkg->iostat.last);
1011
-
percpu_ref_put(&blkg->refcnt);
1012
994
}
1013
-
995
+
raw_spin_unlock(&blkg_stat_lock);
1014
996
out:
1015
997
rcu_read_unlock();
998
+
}
999
+
1000
+
static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
1001
+
{
1002
+
/* Root-level stats are sourced from system-wide IO stats */
1003
+
if (cgroup_parent(css->cgroup))
1004
+
__blkcg_rstat_flush(css_to_blkcg(css), cpu);
1016
1005
}
1017
1006
1018
1007
/*
···
2098
2075
2099
2076
llist_add(&bis->lnode, lhead);
2100
2077
WRITE_ONCE(bis->lqueued, true);
2101
-
percpu_ref_get(&bis->blkg->refcnt);
2102
2078
}
2103
2079
2104
2080
u64_stats_update_end_irqrestore(&bis->sync, flags);