Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * include/linux/backing-dev.h
4 *
5 * low-level device information and state which is propagated up through
6 * to high-level code.
7 */
8
9#ifndef _LINUX_BACKING_DEV_H
10#define _LINUX_BACKING_DEV_H
11
12#include <linux/kernel.h>
13#include <linux/fs.h>
14#include <linux/sched.h>
15#include <linux/device.h>
16#include <linux/writeback.h>
17#include <linux/backing-dev-defs.h>
18#include <linux/slab.h>
19
20static inline struct backing_dev_info *bdi_get(struct backing_dev_info *bdi)
21{
22 kref_get(&bdi->refcnt);
23 return bdi;
24}
25
26struct backing_dev_info *bdi_get_by_id(u64 id);
27void bdi_put(struct backing_dev_info *bdi);
28
29__printf(2, 3)
30int bdi_register(struct backing_dev_info *bdi, const char *fmt, ...);
31__printf(2, 0)
32int bdi_register_va(struct backing_dev_info *bdi, const char *fmt,
33 va_list args);
34void bdi_set_owner(struct backing_dev_info *bdi, struct device *owner);
35void bdi_unregister(struct backing_dev_info *bdi);
36
37struct backing_dev_info *bdi_alloc(int node_id);
38
39void wb_start_background_writeback(struct bdi_writeback *wb);
40void wb_workfn(struct work_struct *work);
41
42void wb_wait_for_completion(struct wb_completion *done);
43
44extern spinlock_t bdi_lock;
45extern struct list_head bdi_list;
46
47extern struct workqueue_struct *bdi_wq;
48
49static inline bool wb_has_dirty_io(struct bdi_writeback *wb)
50{
51 return test_bit(WB_has_dirty_io, &wb->state);
52}
53
54static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi)
55{
56 /*
57 * @bdi->tot_write_bandwidth is guaranteed to be > 0 if there are
58 * any dirty wbs. See wb_update_write_bandwidth().
59 */
60 return atomic_long_read(&bdi->tot_write_bandwidth);
61}
62
63static inline void wb_stat_mod(struct bdi_writeback *wb,
64 enum wb_stat_item item, s64 amount)
65{
66 percpu_counter_add_batch(&wb->stat[item], amount, WB_STAT_BATCH);
67}
68
69static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
70{
71 return percpu_counter_read_positive(&wb->stat[item]);
72}
73
74static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item)
75{
76 return percpu_counter_sum_positive(&wb->stat[item]);
77}
78
79extern void wb_writeout_inc(struct bdi_writeback *wb);
80
81/*
82 * maximal error of a stat counter.
83 */
84static inline unsigned long wb_stat_error(void)
85{
86#ifdef CONFIG_SMP
87 return nr_cpu_ids * WB_STAT_BATCH;
88#else
89 return 1;
90#endif
91}
92
93/* BDI ratio is expressed as part per 1000000 for finer granularity. */
94#define BDI_RATIO_SCALE 10000
95
96u64 bdi_get_min_bytes(struct backing_dev_info *bdi);
97u64 bdi_get_max_bytes(struct backing_dev_info *bdi);
98int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio);
99int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
100int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio);
101int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio);
102int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes);
103int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes);
104int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit);
105
106/*
107 * Flags in backing_dev_info::capability
108 *
109 * BDI_CAP_WRITEBACK: Supports dirty page writeback, and dirty pages
110 * should contribute to accounting
111 * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold
112 */
113#define BDI_CAP_WRITEBACK (1 << 0)
114#define BDI_CAP_STRICTLIMIT (1 << 1)
115
116extern struct backing_dev_info noop_backing_dev_info;
117
118int bdi_init(struct backing_dev_info *bdi);
119
120/**
121 * writeback_in_progress - determine whether there is writeback in progress
122 * @wb: bdi_writeback of interest
123 *
124 * Determine whether there is writeback waiting to be handled against a
125 * bdi_writeback.
126 */
127static inline bool writeback_in_progress(struct bdi_writeback *wb)
128{
129 return test_bit(WB_writeback_running, &wb->state);
130}
131
132struct backing_dev_info *inode_to_bdi(struct inode *inode);
133
134static inline bool mapping_can_writeback(struct address_space *mapping)
135{
136 return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK;
137}
138
139/* Must not be used by file systems that support cgroup writeback */
140static inline int bdi_wb_dirty_exceeded(struct backing_dev_info *bdi)
141{
142 return bdi->wb.dirty_exceeded;
143}
144
145/* Must not be used by file systems that support cgroup writeback */
146static inline void bdi_wb_stat_mod(struct inode *inode, enum wb_stat_item item,
147 s64 amount)
148{
149 wb_stat_mod(&inode_to_bdi(inode)->wb, item, amount);
150}
151
152#ifdef CONFIG_CGROUP_WRITEBACK
153
154struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi,
155 struct cgroup_subsys_state *memcg_css);
156struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
157 struct cgroup_subsys_state *memcg_css,
158 gfp_t gfp);
159void wb_memcg_offline(struct mem_cgroup *memcg);
160void wb_blkcg_offline(struct cgroup_subsys_state *css);
161
162/**
163 * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
164 * @inode: inode of interest
165 *
166 * Cgroup writeback requires support from the filesystem. Also, both memcg and
167 * iocg have to be on the default hierarchy. Test whether all conditions are
168 * met.
169 *
170 * Note that the test result may change dynamically on the same inode
171 * depending on how memcg and iocg are configured.
172 */
173static inline bool inode_cgwb_enabled(struct inode *inode)
174{
175 struct backing_dev_info *bdi = inode_to_bdi(inode);
176
177 return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
178 cgroup_subsys_on_dfl(io_cgrp_subsys) &&
179 (bdi->capabilities & BDI_CAP_WRITEBACK) &&
180 (inode->i_sb->s_iflags & SB_I_CGROUPWB);
181}
182
183/**
184 * wb_find_current - find wb for %current on a bdi
185 * @bdi: bdi of interest
186 *
187 * Find the wb of @bdi which matches both the memcg and blkcg of %current.
188 * Must be called under rcu_read_lock() which protects the returend wb.
189 * NULL if not found.
190 */
191static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
192{
193 struct cgroup_subsys_state *memcg_css;
194 struct bdi_writeback *wb;
195
196 memcg_css = task_css(current, memory_cgrp_id);
197 if (!memcg_css->parent)
198 return &bdi->wb;
199
200 wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
201
202 /*
203 * %current's blkcg equals the effective blkcg of its memcg. No
204 * need to use the relatively expensive cgroup_get_e_css().
205 */
206 if (likely(wb && wb->blkcg_css == task_css(current, io_cgrp_id)))
207 return wb;
208 return NULL;
209}
210
211/**
212 * wb_get_create_current - get or create wb for %current on a bdi
213 * @bdi: bdi of interest
214 * @gfp: allocation mask
215 *
216 * Equivalent to wb_get_create() on %current's memcg. This function is
217 * called from a relatively hot path and optimizes the common cases using
218 * wb_find_current().
219 */
220static inline struct bdi_writeback *
221wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
222{
223 struct bdi_writeback *wb;
224
225 rcu_read_lock();
226 wb = wb_find_current(bdi);
227 if (wb && unlikely(!wb_tryget(wb)))
228 wb = NULL;
229 rcu_read_unlock();
230
231 if (unlikely(!wb)) {
232 struct cgroup_subsys_state *memcg_css;
233
234 memcg_css = task_get_css(current, memory_cgrp_id);
235 wb = wb_get_create(bdi, memcg_css, gfp);
236 css_put(memcg_css);
237 }
238 return wb;
239}
240
241/**
242 * inode_to_wb - determine the wb of an inode
243 * @inode: inode of interest
244 *
245 * Returns the wb @inode is currently associated with. The caller must be
246 * holding either @inode->i_lock, the i_pages lock, or the
247 * associated wb's list_lock.
248 */
249static inline struct bdi_writeback *inode_to_wb(const struct inode *inode)
250{
251#ifdef CONFIG_LOCKDEP
252 WARN_ON_ONCE(debug_locks &&
253 (inode->i_sb->s_iflags & SB_I_CGROUPWB) &&
254 (!lockdep_is_held(&inode->i_lock) &&
255 !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) &&
256 !lockdep_is_held(&inode->i_wb->list_lock)));
257#endif
258 return inode->i_wb;
259}
260
261static inline struct bdi_writeback *inode_to_wb_wbc(
262 struct inode *inode,
263 struct writeback_control *wbc)
264{
265 /*
266 * If wbc does not have inode attached, it means cgroup writeback was
267 * disabled when wbc started. Just use the default wb in that case.
268 */
269 return wbc->wb ? wbc->wb : &inode_to_bdi(inode)->wb;
270}
271
272/**
273 * unlocked_inode_to_wb_begin - begin unlocked inode wb access transaction
274 * @inode: target inode
275 * @cookie: output param, to be passed to the end function
276 *
277 * The caller wants to access the wb associated with @inode but isn't
278 * holding inode->i_lock, the i_pages lock or wb->list_lock. This
279 * function determines the wb associated with @inode and ensures that the
280 * association doesn't change until the transaction is finished with
281 * unlocked_inode_to_wb_end().
282 *
283 * The caller must call unlocked_inode_to_wb_end() with *@cookie afterwards and
284 * can't sleep during the transaction. IRQs may or may not be disabled on
285 * return.
286 */
287static inline struct bdi_writeback *
288unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
289{
290 rcu_read_lock();
291
292 /*
293 * Paired with a release fence in inode_do_switch_wbs() and
294 * ensures that we see the new wb if we see cleared I_WB_SWITCH.
295 */
296 cookie->locked = inode_state_read_once(inode) & I_WB_SWITCH;
297 smp_rmb();
298
299 if (unlikely(cookie->locked))
300 xa_lock_irqsave(&inode->i_mapping->i_pages, cookie->flags);
301
302 /*
303 * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages
304 * lock. inode_to_wb() will bark. Deref directly.
305 */
306 return inode->i_wb;
307}
308
309/**
310 * unlocked_inode_to_wb_end - end inode wb access transaction
311 * @inode: target inode
312 * @cookie: @cookie from unlocked_inode_to_wb_begin()
313 */
314static inline void unlocked_inode_to_wb_end(struct inode *inode,
315 struct wb_lock_cookie *cookie)
316{
317 if (unlikely(cookie->locked))
318 xa_unlock_irqrestore(&inode->i_mapping->i_pages, cookie->flags);
319
320 rcu_read_unlock();
321}
322
323#else /* CONFIG_CGROUP_WRITEBACK */
324
325static inline bool inode_cgwb_enabled(struct inode *inode)
326{
327 return false;
328}
329
330static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
331{
332 return &bdi->wb;
333}
334
335static inline struct bdi_writeback *
336wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
337{
338 return &bdi->wb;
339}
340
341static inline struct bdi_writeback *inode_to_wb(struct inode *inode)
342{
343 return &inode_to_bdi(inode)->wb;
344}
345
346static inline struct bdi_writeback *inode_to_wb_wbc(
347 struct inode *inode,
348 struct writeback_control *wbc)
349{
350 return inode_to_wb(inode);
351}
352
353
354static inline struct bdi_writeback *
355unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
356{
357 return inode_to_wb(inode);
358}
359
360static inline void unlocked_inode_to_wb_end(struct inode *inode,
361 struct wb_lock_cookie *cookie)
362{
363}
364
365static inline void wb_memcg_offline(struct mem_cgroup *memcg)
366{
367}
368
369static inline void wb_blkcg_offline(struct cgroup_subsys_state *css)
370{
371}
372
373#endif /* CONFIG_CGROUP_WRITEBACK */
374
375const char *bdi_dev_name(struct backing_dev_info *bdi);
376
377#endif /* _LINUX_BACKING_DEV_H */